Upload bench/base_full/results_2026-07-16T13-49-47.872447.json with huggingface_hub
Browse files
bench/base_full/results_2026-07-16T13-49-47.872447.json
ADDED
|
@@ -0,0 +1,1377 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"results": {
|
| 3 |
+
"mmlu_pro_biology": {
|
| 4 |
+
"name": "mmlu_pro_biology",
|
| 5 |
+
"alias": "biology",
|
| 6 |
+
"sample_len": 25,
|
| 7 |
+
"exact_match,custom-extract": 0.6,
|
| 8 |
+
"exact_match_stderr,custom-extract": 0.10000000000000002
|
| 9 |
+
},
|
| 10 |
+
"mmlu_pro_business": {
|
| 11 |
+
"name": "mmlu_pro_business",
|
| 12 |
+
"alias": "business",
|
| 13 |
+
"sample_len": 25,
|
| 14 |
+
"exact_match,custom-extract": 0.24,
|
| 15 |
+
"exact_match_stderr,custom-extract": 0.08717797887081347
|
| 16 |
+
},
|
| 17 |
+
"mmlu_pro_chemistry": {
|
| 18 |
+
"name": "mmlu_pro_chemistry",
|
| 19 |
+
"alias": "chemistry",
|
| 20 |
+
"sample_len": 25,
|
| 21 |
+
"exact_match,custom-extract": 0.64,
|
| 22 |
+
"exact_match_stderr,custom-extract": 0.09797958971132711
|
| 23 |
+
},
|
| 24 |
+
"mmlu_pro_computer_science": {
|
| 25 |
+
"name": "mmlu_pro_computer_science",
|
| 26 |
+
"alias": "computer_science",
|
| 27 |
+
"sample_len": 25,
|
| 28 |
+
"exact_match,custom-extract": 0.44,
|
| 29 |
+
"exact_match_stderr,custom-extract": 0.10132456102380442
|
| 30 |
+
},
|
| 31 |
+
"mmlu_pro_economics": {
|
| 32 |
+
"name": "mmlu_pro_economics",
|
| 33 |
+
"alias": "economics",
|
| 34 |
+
"sample_len": 25,
|
| 35 |
+
"exact_match,custom-extract": 0.28,
|
| 36 |
+
"exact_match_stderr,custom-extract": 0.09165151389911677
|
| 37 |
+
},
|
| 38 |
+
"mmlu_pro_engineering": {
|
| 39 |
+
"name": "mmlu_pro_engineering",
|
| 40 |
+
"alias": "engineering",
|
| 41 |
+
"sample_len": 25,
|
| 42 |
+
"exact_match,custom-extract": 0.32,
|
| 43 |
+
"exact_match_stderr,custom-extract": 0.09521904571390465
|
| 44 |
+
},
|
| 45 |
+
"mmlu_pro_health": {
|
| 46 |
+
"name": "mmlu_pro_health",
|
| 47 |
+
"alias": "health",
|
| 48 |
+
"sample_len": 25,
|
| 49 |
+
"exact_match,custom-extract": 0.32,
|
| 50 |
+
"exact_match_stderr,custom-extract": 0.09521904571390466
|
| 51 |
+
},
|
| 52 |
+
"mmlu_pro_history": {
|
| 53 |
+
"name": "mmlu_pro_history",
|
| 54 |
+
"alias": "history",
|
| 55 |
+
"sample_len": 25,
|
| 56 |
+
"exact_match,custom-extract": 0.32,
|
| 57 |
+
"exact_match_stderr,custom-extract": 0.09521904571390466
|
| 58 |
+
},
|
| 59 |
+
"mmlu_pro_law": {
|
| 60 |
+
"name": "mmlu_pro_law",
|
| 61 |
+
"alias": "law",
|
| 62 |
+
"sample_len": 25,
|
| 63 |
+
"exact_match,custom-extract": 0.2,
|
| 64 |
+
"exact_match_stderr,custom-extract": 0.08164965809277261
|
| 65 |
+
},
|
| 66 |
+
"mmlu_pro_math": {
|
| 67 |
+
"name": "mmlu_pro_math",
|
| 68 |
+
"alias": "math",
|
| 69 |
+
"sample_len": 25,
|
| 70 |
+
"exact_match,custom-extract": 0.56,
|
| 71 |
+
"exact_match_stderr,custom-extract": 0.10132456102380442
|
| 72 |
+
},
|
| 73 |
+
"mmlu_pro_other": {
|
| 74 |
+
"name": "mmlu_pro_other",
|
| 75 |
+
"alias": "other",
|
| 76 |
+
"sample_len": 25,
|
| 77 |
+
"exact_match,custom-extract": 0.24,
|
| 78 |
+
"exact_match_stderr,custom-extract": 0.08717797887081345
|
| 79 |
+
},
|
| 80 |
+
"mmlu_pro_philosophy": {
|
| 81 |
+
"name": "mmlu_pro_philosophy",
|
| 82 |
+
"alias": "philosophy",
|
| 83 |
+
"sample_len": 25,
|
| 84 |
+
"exact_match,custom-extract": 0.28,
|
| 85 |
+
"exact_match_stderr,custom-extract": 0.09165151389911677
|
| 86 |
+
},
|
| 87 |
+
"mmlu_pro_physics": {
|
| 88 |
+
"name": "mmlu_pro_physics",
|
| 89 |
+
"alias": "physics",
|
| 90 |
+
"sample_len": 25,
|
| 91 |
+
"exact_match,custom-extract": 0.56,
|
| 92 |
+
"exact_match_stderr,custom-extract": 0.10132456102380442
|
| 93 |
+
},
|
| 94 |
+
"mmlu_pro_psychology": {
|
| 95 |
+
"name": "mmlu_pro_psychology",
|
| 96 |
+
"alias": "psychology",
|
| 97 |
+
"sample_len": 25,
|
| 98 |
+
"exact_match,custom-extract": 0.08,
|
| 99 |
+
"exact_match_stderr,custom-extract": 0.05537749241945382
|
| 100 |
+
},
|
| 101 |
+
"mmlu_pro": {
|
| 102 |
+
"alias": "mmlu_pro",
|
| 103 |
+
"name": "mmlu_pro",
|
| 104 |
+
"sample_len": 350,
|
| 105 |
+
"exact_match,custom-extract": 0.3628571428571429,
|
| 106 |
+
"exact_match_stderr,custom-extract": 0.02467475506291736,
|
| 107 |
+
"sample_count": {
|
| 108 |
+
"exact_match,custom-extract": 350
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
},
|
| 112 |
+
"groups": {
|
| 113 |
+
"mmlu_pro": {
|
| 114 |
+
"alias": "mmlu_pro",
|
| 115 |
+
"name": "mmlu_pro",
|
| 116 |
+
"sample_len": 350,
|
| 117 |
+
"exact_match,custom-extract": 0.3628571428571429,
|
| 118 |
+
"exact_match_stderr,custom-extract": 0.02467475506291736,
|
| 119 |
+
"sample_count": {
|
| 120 |
+
"exact_match,custom-extract": 350
|
| 121 |
+
}
|
| 122 |
+
}
|
| 123 |
+
},
|
| 124 |
+
"group_subtasks": {
|
| 125 |
+
"mmlu_pro": [
|
| 126 |
+
"mmlu_pro_biology",
|
| 127 |
+
"mmlu_pro_business",
|
| 128 |
+
"mmlu_pro_chemistry",
|
| 129 |
+
"mmlu_pro_computer_science",
|
| 130 |
+
"mmlu_pro_economics",
|
| 131 |
+
"mmlu_pro_engineering",
|
| 132 |
+
"mmlu_pro_health",
|
| 133 |
+
"mmlu_pro_history",
|
| 134 |
+
"mmlu_pro_law",
|
| 135 |
+
"mmlu_pro_math",
|
| 136 |
+
"mmlu_pro_other",
|
| 137 |
+
"mmlu_pro_philosophy",
|
| 138 |
+
"mmlu_pro_physics",
|
| 139 |
+
"mmlu_pro_psychology"
|
| 140 |
+
]
|
| 141 |
+
},
|
| 142 |
+
"configs": {
|
| 143 |
+
"mmlu_pro_biology": {
|
| 144 |
+
"task": "mmlu_pro_biology",
|
| 145 |
+
"task_alias": "biology",
|
| 146 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 147 |
+
"test_split": "test",
|
| 148 |
+
"fewshot_split": "validation",
|
| 149 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='biology')",
|
| 150 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 151 |
+
"doc_to_target": "answer",
|
| 152 |
+
"unsafe_code": false,
|
| 153 |
+
"description": "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 154 |
+
"target_delimiter": " ",
|
| 155 |
+
"fewshot_delimiter": "\n\n",
|
| 156 |
+
"fewshot_config": {
|
| 157 |
+
"sampler": "first_n",
|
| 158 |
+
"split": "validation",
|
| 159 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='biology')",
|
| 160 |
+
"fewshot_indices": null,
|
| 161 |
+
"samples": null,
|
| 162 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 163 |
+
"doc_to_choice": null,
|
| 164 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 165 |
+
"gen_prefix": null,
|
| 166 |
+
"fewshot_delimiter": "\n\n",
|
| 167 |
+
"target_delimiter": " "
|
| 168 |
+
},
|
| 169 |
+
"num_fewshot": 5,
|
| 170 |
+
"metric_list": [
|
| 171 |
+
{
|
| 172 |
+
"metric": "exact_match",
|
| 173 |
+
"aggregation": "mean",
|
| 174 |
+
"higher_is_better": true,
|
| 175 |
+
"ignore_case": true,
|
| 176 |
+
"ignore_punctuation": true
|
| 177 |
+
}
|
| 178 |
+
],
|
| 179 |
+
"output_type": "generate_until",
|
| 180 |
+
"generation_kwargs": {
|
| 181 |
+
"until": [
|
| 182 |
+
"Question:"
|
| 183 |
+
],
|
| 184 |
+
"max_gen_toks": 32768,
|
| 185 |
+
"do_sample": false,
|
| 186 |
+
"temperature": 1.0,
|
| 187 |
+
"top_p": 0.95,
|
| 188 |
+
"presence_penalty": 1.5
|
| 189 |
+
},
|
| 190 |
+
"repeats": 1,
|
| 191 |
+
"filter_list": [
|
| 192 |
+
{
|
| 193 |
+
"name": "custom-extract",
|
| 194 |
+
"filter": [
|
| 195 |
+
{
|
| 196 |
+
"function": "regex",
|
| 197 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"function": "take_first"
|
| 201 |
+
}
|
| 202 |
+
]
|
| 203 |
+
}
|
| 204 |
+
],
|
| 205 |
+
"should_decontaminate": false,
|
| 206 |
+
"metadata": {
|
| 207 |
+
"version": 3.1,
|
| 208 |
+
"model": "qwen35-4b-base",
|
| 209 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 210 |
+
"num_concurrent": 16,
|
| 211 |
+
"max_retries": 3,
|
| 212 |
+
"timeout": 1800,
|
| 213 |
+
"tokenized_requests": false,
|
| 214 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml"
|
| 215 |
+
}
|
| 216 |
+
},
|
| 217 |
+
"mmlu_pro_business": {
|
| 218 |
+
"task": "mmlu_pro_business",
|
| 219 |
+
"task_alias": "business",
|
| 220 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 221 |
+
"test_split": "test",
|
| 222 |
+
"fewshot_split": "validation",
|
| 223 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='business')",
|
| 224 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 225 |
+
"doc_to_target": "answer",
|
| 226 |
+
"unsafe_code": false,
|
| 227 |
+
"description": "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 228 |
+
"target_delimiter": " ",
|
| 229 |
+
"fewshot_delimiter": "\n\n",
|
| 230 |
+
"fewshot_config": {
|
| 231 |
+
"sampler": "first_n",
|
| 232 |
+
"split": "validation",
|
| 233 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='business')",
|
| 234 |
+
"fewshot_indices": null,
|
| 235 |
+
"samples": null,
|
| 236 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 237 |
+
"doc_to_choice": null,
|
| 238 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 239 |
+
"gen_prefix": null,
|
| 240 |
+
"fewshot_delimiter": "\n\n",
|
| 241 |
+
"target_delimiter": " "
|
| 242 |
+
},
|
| 243 |
+
"num_fewshot": 5,
|
| 244 |
+
"metric_list": [
|
| 245 |
+
{
|
| 246 |
+
"metric": "exact_match",
|
| 247 |
+
"aggregation": "mean",
|
| 248 |
+
"higher_is_better": true,
|
| 249 |
+
"ignore_case": true,
|
| 250 |
+
"ignore_punctuation": true
|
| 251 |
+
}
|
| 252 |
+
],
|
| 253 |
+
"output_type": "generate_until",
|
| 254 |
+
"generation_kwargs": {
|
| 255 |
+
"until": [
|
| 256 |
+
"Question:"
|
| 257 |
+
],
|
| 258 |
+
"max_gen_toks": 32768,
|
| 259 |
+
"do_sample": false,
|
| 260 |
+
"temperature": 1.0,
|
| 261 |
+
"top_p": 0.95,
|
| 262 |
+
"presence_penalty": 1.5
|
| 263 |
+
},
|
| 264 |
+
"repeats": 1,
|
| 265 |
+
"filter_list": [
|
| 266 |
+
{
|
| 267 |
+
"name": "custom-extract",
|
| 268 |
+
"filter": [
|
| 269 |
+
{
|
| 270 |
+
"function": "regex",
|
| 271 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 272 |
+
},
|
| 273 |
+
{
|
| 274 |
+
"function": "take_first"
|
| 275 |
+
}
|
| 276 |
+
]
|
| 277 |
+
}
|
| 278 |
+
],
|
| 279 |
+
"should_decontaminate": false,
|
| 280 |
+
"metadata": {
|
| 281 |
+
"version": 3.1,
|
| 282 |
+
"model": "qwen35-4b-base",
|
| 283 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 284 |
+
"num_concurrent": 16,
|
| 285 |
+
"max_retries": 3,
|
| 286 |
+
"timeout": 1800,
|
| 287 |
+
"tokenized_requests": false,
|
| 288 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml"
|
| 289 |
+
}
|
| 290 |
+
},
|
| 291 |
+
"mmlu_pro_chemistry": {
|
| 292 |
+
"task": "mmlu_pro_chemistry",
|
| 293 |
+
"task_alias": "chemistry",
|
| 294 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 295 |
+
"test_split": "test",
|
| 296 |
+
"fewshot_split": "validation",
|
| 297 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='chemistry')",
|
| 298 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 299 |
+
"doc_to_target": "answer",
|
| 300 |
+
"unsafe_code": false,
|
| 301 |
+
"description": "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 302 |
+
"target_delimiter": " ",
|
| 303 |
+
"fewshot_delimiter": "\n\n",
|
| 304 |
+
"fewshot_config": {
|
| 305 |
+
"sampler": "first_n",
|
| 306 |
+
"split": "validation",
|
| 307 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='chemistry')",
|
| 308 |
+
"fewshot_indices": null,
|
| 309 |
+
"samples": null,
|
| 310 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 311 |
+
"doc_to_choice": null,
|
| 312 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 313 |
+
"gen_prefix": null,
|
| 314 |
+
"fewshot_delimiter": "\n\n",
|
| 315 |
+
"target_delimiter": " "
|
| 316 |
+
},
|
| 317 |
+
"num_fewshot": 5,
|
| 318 |
+
"metric_list": [
|
| 319 |
+
{
|
| 320 |
+
"metric": "exact_match",
|
| 321 |
+
"aggregation": "mean",
|
| 322 |
+
"higher_is_better": true,
|
| 323 |
+
"ignore_case": true,
|
| 324 |
+
"ignore_punctuation": true
|
| 325 |
+
}
|
| 326 |
+
],
|
| 327 |
+
"output_type": "generate_until",
|
| 328 |
+
"generation_kwargs": {
|
| 329 |
+
"until": [
|
| 330 |
+
"Question:"
|
| 331 |
+
],
|
| 332 |
+
"max_gen_toks": 32768,
|
| 333 |
+
"do_sample": false,
|
| 334 |
+
"temperature": 1.0,
|
| 335 |
+
"top_p": 0.95,
|
| 336 |
+
"presence_penalty": 1.5
|
| 337 |
+
},
|
| 338 |
+
"repeats": 1,
|
| 339 |
+
"filter_list": [
|
| 340 |
+
{
|
| 341 |
+
"name": "custom-extract",
|
| 342 |
+
"filter": [
|
| 343 |
+
{
|
| 344 |
+
"function": "regex",
|
| 345 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 346 |
+
},
|
| 347 |
+
{
|
| 348 |
+
"function": "take_first"
|
| 349 |
+
}
|
| 350 |
+
]
|
| 351 |
+
}
|
| 352 |
+
],
|
| 353 |
+
"should_decontaminate": false,
|
| 354 |
+
"metadata": {
|
| 355 |
+
"version": 3.1,
|
| 356 |
+
"model": "qwen35-4b-base",
|
| 357 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 358 |
+
"num_concurrent": 16,
|
| 359 |
+
"max_retries": 3,
|
| 360 |
+
"timeout": 1800,
|
| 361 |
+
"tokenized_requests": false,
|
| 362 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml"
|
| 363 |
+
}
|
| 364 |
+
},
|
| 365 |
+
"mmlu_pro_computer_science": {
|
| 366 |
+
"task": "mmlu_pro_computer_science",
|
| 367 |
+
"task_alias": "computer_science",
|
| 368 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 369 |
+
"test_split": "test",
|
| 370 |
+
"fewshot_split": "validation",
|
| 371 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='computer science')",
|
| 372 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 373 |
+
"doc_to_target": "answer",
|
| 374 |
+
"unsafe_code": false,
|
| 375 |
+
"description": "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 376 |
+
"target_delimiter": " ",
|
| 377 |
+
"fewshot_delimiter": "\n\n",
|
| 378 |
+
"fewshot_config": {
|
| 379 |
+
"sampler": "first_n",
|
| 380 |
+
"split": "validation",
|
| 381 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='computer science')",
|
| 382 |
+
"fewshot_indices": null,
|
| 383 |
+
"samples": null,
|
| 384 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 385 |
+
"doc_to_choice": null,
|
| 386 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 387 |
+
"gen_prefix": null,
|
| 388 |
+
"fewshot_delimiter": "\n\n",
|
| 389 |
+
"target_delimiter": " "
|
| 390 |
+
},
|
| 391 |
+
"num_fewshot": 5,
|
| 392 |
+
"metric_list": [
|
| 393 |
+
{
|
| 394 |
+
"metric": "exact_match",
|
| 395 |
+
"aggregation": "mean",
|
| 396 |
+
"higher_is_better": true,
|
| 397 |
+
"ignore_case": true,
|
| 398 |
+
"ignore_punctuation": true
|
| 399 |
+
}
|
| 400 |
+
],
|
| 401 |
+
"output_type": "generate_until",
|
| 402 |
+
"generation_kwargs": {
|
| 403 |
+
"until": [
|
| 404 |
+
"Question:"
|
| 405 |
+
],
|
| 406 |
+
"max_gen_toks": 32768,
|
| 407 |
+
"do_sample": false,
|
| 408 |
+
"temperature": 1.0,
|
| 409 |
+
"top_p": 0.95,
|
| 410 |
+
"presence_penalty": 1.5
|
| 411 |
+
},
|
| 412 |
+
"repeats": 1,
|
| 413 |
+
"filter_list": [
|
| 414 |
+
{
|
| 415 |
+
"name": "custom-extract",
|
| 416 |
+
"filter": [
|
| 417 |
+
{
|
| 418 |
+
"function": "regex",
|
| 419 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"function": "take_first"
|
| 423 |
+
}
|
| 424 |
+
]
|
| 425 |
+
}
|
| 426 |
+
],
|
| 427 |
+
"should_decontaminate": false,
|
| 428 |
+
"metadata": {
|
| 429 |
+
"version": 3.1,
|
| 430 |
+
"model": "qwen35-4b-base",
|
| 431 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 432 |
+
"num_concurrent": 16,
|
| 433 |
+
"max_retries": 3,
|
| 434 |
+
"timeout": 1800,
|
| 435 |
+
"tokenized_requests": false,
|
| 436 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml"
|
| 437 |
+
}
|
| 438 |
+
},
|
| 439 |
+
"mmlu_pro_economics": {
|
| 440 |
+
"task": "mmlu_pro_economics",
|
| 441 |
+
"task_alias": "economics",
|
| 442 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 443 |
+
"test_split": "test",
|
| 444 |
+
"fewshot_split": "validation",
|
| 445 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='economics')",
|
| 446 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 447 |
+
"doc_to_target": "answer",
|
| 448 |
+
"unsafe_code": false,
|
| 449 |
+
"description": "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 450 |
+
"target_delimiter": " ",
|
| 451 |
+
"fewshot_delimiter": "\n\n",
|
| 452 |
+
"fewshot_config": {
|
| 453 |
+
"sampler": "first_n",
|
| 454 |
+
"split": "validation",
|
| 455 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='economics')",
|
| 456 |
+
"fewshot_indices": null,
|
| 457 |
+
"samples": null,
|
| 458 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 459 |
+
"doc_to_choice": null,
|
| 460 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 461 |
+
"gen_prefix": null,
|
| 462 |
+
"fewshot_delimiter": "\n\n",
|
| 463 |
+
"target_delimiter": " "
|
| 464 |
+
},
|
| 465 |
+
"num_fewshot": 5,
|
| 466 |
+
"metric_list": [
|
| 467 |
+
{
|
| 468 |
+
"metric": "exact_match",
|
| 469 |
+
"aggregation": "mean",
|
| 470 |
+
"higher_is_better": true,
|
| 471 |
+
"ignore_case": true,
|
| 472 |
+
"ignore_punctuation": true
|
| 473 |
+
}
|
| 474 |
+
],
|
| 475 |
+
"output_type": "generate_until",
|
| 476 |
+
"generation_kwargs": {
|
| 477 |
+
"until": [
|
| 478 |
+
"Question:"
|
| 479 |
+
],
|
| 480 |
+
"max_gen_toks": 32768,
|
| 481 |
+
"do_sample": false,
|
| 482 |
+
"temperature": 1.0,
|
| 483 |
+
"top_p": 0.95,
|
| 484 |
+
"presence_penalty": 1.5
|
| 485 |
+
},
|
| 486 |
+
"repeats": 1,
|
| 487 |
+
"filter_list": [
|
| 488 |
+
{
|
| 489 |
+
"name": "custom-extract",
|
| 490 |
+
"filter": [
|
| 491 |
+
{
|
| 492 |
+
"function": "regex",
|
| 493 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"function": "take_first"
|
| 497 |
+
}
|
| 498 |
+
]
|
| 499 |
+
}
|
| 500 |
+
],
|
| 501 |
+
"should_decontaminate": false,
|
| 502 |
+
"metadata": {
|
| 503 |
+
"version": 3.1,
|
| 504 |
+
"model": "qwen35-4b-base",
|
| 505 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 506 |
+
"num_concurrent": 16,
|
| 507 |
+
"max_retries": 3,
|
| 508 |
+
"timeout": 1800,
|
| 509 |
+
"tokenized_requests": false,
|
| 510 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml"
|
| 511 |
+
}
|
| 512 |
+
},
|
| 513 |
+
"mmlu_pro_engineering": {
|
| 514 |
+
"task": "mmlu_pro_engineering",
|
| 515 |
+
"task_alias": "engineering",
|
| 516 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 517 |
+
"test_split": "test",
|
| 518 |
+
"fewshot_split": "validation",
|
| 519 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='engineering')",
|
| 520 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 521 |
+
"doc_to_target": "answer",
|
| 522 |
+
"unsafe_code": false,
|
| 523 |
+
"description": "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 524 |
+
"target_delimiter": " ",
|
| 525 |
+
"fewshot_delimiter": "\n\n",
|
| 526 |
+
"fewshot_config": {
|
| 527 |
+
"sampler": "first_n",
|
| 528 |
+
"split": "validation",
|
| 529 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='engineering')",
|
| 530 |
+
"fewshot_indices": null,
|
| 531 |
+
"samples": null,
|
| 532 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 533 |
+
"doc_to_choice": null,
|
| 534 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 535 |
+
"gen_prefix": null,
|
| 536 |
+
"fewshot_delimiter": "\n\n",
|
| 537 |
+
"target_delimiter": " "
|
| 538 |
+
},
|
| 539 |
+
"num_fewshot": 5,
|
| 540 |
+
"metric_list": [
|
| 541 |
+
{
|
| 542 |
+
"metric": "exact_match",
|
| 543 |
+
"aggregation": "mean",
|
| 544 |
+
"higher_is_better": true,
|
| 545 |
+
"ignore_case": true,
|
| 546 |
+
"ignore_punctuation": true
|
| 547 |
+
}
|
| 548 |
+
],
|
| 549 |
+
"output_type": "generate_until",
|
| 550 |
+
"generation_kwargs": {
|
| 551 |
+
"until": [
|
| 552 |
+
"Question:"
|
| 553 |
+
],
|
| 554 |
+
"max_gen_toks": 32768,
|
| 555 |
+
"do_sample": false,
|
| 556 |
+
"temperature": 1.0,
|
| 557 |
+
"top_p": 0.95,
|
| 558 |
+
"presence_penalty": 1.5
|
| 559 |
+
},
|
| 560 |
+
"repeats": 1,
|
| 561 |
+
"filter_list": [
|
| 562 |
+
{
|
| 563 |
+
"name": "custom-extract",
|
| 564 |
+
"filter": [
|
| 565 |
+
{
|
| 566 |
+
"function": "regex",
|
| 567 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"function": "take_first"
|
| 571 |
+
}
|
| 572 |
+
]
|
| 573 |
+
}
|
| 574 |
+
],
|
| 575 |
+
"should_decontaminate": false,
|
| 576 |
+
"metadata": {
|
| 577 |
+
"version": 3.1,
|
| 578 |
+
"model": "qwen35-4b-base",
|
| 579 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 580 |
+
"num_concurrent": 16,
|
| 581 |
+
"max_retries": 3,
|
| 582 |
+
"timeout": 1800,
|
| 583 |
+
"tokenized_requests": false,
|
| 584 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml"
|
| 585 |
+
}
|
| 586 |
+
},
|
| 587 |
+
"mmlu_pro_health": {
|
| 588 |
+
"task": "mmlu_pro_health",
|
| 589 |
+
"task_alias": "health",
|
| 590 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 591 |
+
"test_split": "test",
|
| 592 |
+
"fewshot_split": "validation",
|
| 593 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='health')",
|
| 594 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 595 |
+
"doc_to_target": "answer",
|
| 596 |
+
"unsafe_code": false,
|
| 597 |
+
"description": "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 598 |
+
"target_delimiter": " ",
|
| 599 |
+
"fewshot_delimiter": "\n\n",
|
| 600 |
+
"fewshot_config": {
|
| 601 |
+
"sampler": "first_n",
|
| 602 |
+
"split": "validation",
|
| 603 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='health')",
|
| 604 |
+
"fewshot_indices": null,
|
| 605 |
+
"samples": null,
|
| 606 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 607 |
+
"doc_to_choice": null,
|
| 608 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 609 |
+
"gen_prefix": null,
|
| 610 |
+
"fewshot_delimiter": "\n\n",
|
| 611 |
+
"target_delimiter": " "
|
| 612 |
+
},
|
| 613 |
+
"num_fewshot": 5,
|
| 614 |
+
"metric_list": [
|
| 615 |
+
{
|
| 616 |
+
"metric": "exact_match",
|
| 617 |
+
"aggregation": "mean",
|
| 618 |
+
"higher_is_better": true,
|
| 619 |
+
"ignore_case": true,
|
| 620 |
+
"ignore_punctuation": true
|
| 621 |
+
}
|
| 622 |
+
],
|
| 623 |
+
"output_type": "generate_until",
|
| 624 |
+
"generation_kwargs": {
|
| 625 |
+
"until": [
|
| 626 |
+
"Question:"
|
| 627 |
+
],
|
| 628 |
+
"max_gen_toks": 32768,
|
| 629 |
+
"do_sample": false,
|
| 630 |
+
"temperature": 1.0,
|
| 631 |
+
"top_p": 0.95,
|
| 632 |
+
"presence_penalty": 1.5
|
| 633 |
+
},
|
| 634 |
+
"repeats": 1,
|
| 635 |
+
"filter_list": [
|
| 636 |
+
{
|
| 637 |
+
"name": "custom-extract",
|
| 638 |
+
"filter": [
|
| 639 |
+
{
|
| 640 |
+
"function": "regex",
|
| 641 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 642 |
+
},
|
| 643 |
+
{
|
| 644 |
+
"function": "take_first"
|
| 645 |
+
}
|
| 646 |
+
]
|
| 647 |
+
}
|
| 648 |
+
],
|
| 649 |
+
"should_decontaminate": false,
|
| 650 |
+
"metadata": {
|
| 651 |
+
"version": 3.1,
|
| 652 |
+
"model": "qwen35-4b-base",
|
| 653 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 654 |
+
"num_concurrent": 16,
|
| 655 |
+
"max_retries": 3,
|
| 656 |
+
"timeout": 1800,
|
| 657 |
+
"tokenized_requests": false,
|
| 658 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml"
|
| 659 |
+
}
|
| 660 |
+
},
|
| 661 |
+
"mmlu_pro_history": {
|
| 662 |
+
"task": "mmlu_pro_history",
|
| 663 |
+
"task_alias": "history",
|
| 664 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 665 |
+
"test_split": "test",
|
| 666 |
+
"fewshot_split": "validation",
|
| 667 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='history')",
|
| 668 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 669 |
+
"doc_to_target": "answer",
|
| 670 |
+
"unsafe_code": false,
|
| 671 |
+
"description": "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 672 |
+
"target_delimiter": " ",
|
| 673 |
+
"fewshot_delimiter": "\n\n",
|
| 674 |
+
"fewshot_config": {
|
| 675 |
+
"sampler": "first_n",
|
| 676 |
+
"split": "validation",
|
| 677 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='history')",
|
| 678 |
+
"fewshot_indices": null,
|
| 679 |
+
"samples": null,
|
| 680 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 681 |
+
"doc_to_choice": null,
|
| 682 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 683 |
+
"gen_prefix": null,
|
| 684 |
+
"fewshot_delimiter": "\n\n",
|
| 685 |
+
"target_delimiter": " "
|
| 686 |
+
},
|
| 687 |
+
"num_fewshot": 5,
|
| 688 |
+
"metric_list": [
|
| 689 |
+
{
|
| 690 |
+
"metric": "exact_match",
|
| 691 |
+
"aggregation": "mean",
|
| 692 |
+
"higher_is_better": true,
|
| 693 |
+
"ignore_case": true,
|
| 694 |
+
"ignore_punctuation": true
|
| 695 |
+
}
|
| 696 |
+
],
|
| 697 |
+
"output_type": "generate_until",
|
| 698 |
+
"generation_kwargs": {
|
| 699 |
+
"until": [
|
| 700 |
+
"Question:"
|
| 701 |
+
],
|
| 702 |
+
"max_gen_toks": 32768,
|
| 703 |
+
"do_sample": false,
|
| 704 |
+
"temperature": 1.0,
|
| 705 |
+
"top_p": 0.95,
|
| 706 |
+
"presence_penalty": 1.5
|
| 707 |
+
},
|
| 708 |
+
"repeats": 1,
|
| 709 |
+
"filter_list": [
|
| 710 |
+
{
|
| 711 |
+
"name": "custom-extract",
|
| 712 |
+
"filter": [
|
| 713 |
+
{
|
| 714 |
+
"function": "regex",
|
| 715 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 716 |
+
},
|
| 717 |
+
{
|
| 718 |
+
"function": "take_first"
|
| 719 |
+
}
|
| 720 |
+
]
|
| 721 |
+
}
|
| 722 |
+
],
|
| 723 |
+
"should_decontaminate": false,
|
| 724 |
+
"metadata": {
|
| 725 |
+
"version": 3.1,
|
| 726 |
+
"model": "qwen35-4b-base",
|
| 727 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 728 |
+
"num_concurrent": 16,
|
| 729 |
+
"max_retries": 3,
|
| 730 |
+
"timeout": 1800,
|
| 731 |
+
"tokenized_requests": false,
|
| 732 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml"
|
| 733 |
+
}
|
| 734 |
+
},
|
| 735 |
+
"mmlu_pro_law": {
|
| 736 |
+
"task": "mmlu_pro_law",
|
| 737 |
+
"task_alias": "law",
|
| 738 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 739 |
+
"test_split": "test",
|
| 740 |
+
"fewshot_split": "validation",
|
| 741 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='law')",
|
| 742 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 743 |
+
"doc_to_target": "answer",
|
| 744 |
+
"unsafe_code": false,
|
| 745 |
+
"description": "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 746 |
+
"target_delimiter": " ",
|
| 747 |
+
"fewshot_delimiter": "\n\n",
|
| 748 |
+
"fewshot_config": {
|
| 749 |
+
"sampler": "first_n",
|
| 750 |
+
"split": "validation",
|
| 751 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='law')",
|
| 752 |
+
"fewshot_indices": null,
|
| 753 |
+
"samples": null,
|
| 754 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 755 |
+
"doc_to_choice": null,
|
| 756 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 757 |
+
"gen_prefix": null,
|
| 758 |
+
"fewshot_delimiter": "\n\n",
|
| 759 |
+
"target_delimiter": " "
|
| 760 |
+
},
|
| 761 |
+
"num_fewshot": 5,
|
| 762 |
+
"metric_list": [
|
| 763 |
+
{
|
| 764 |
+
"metric": "exact_match",
|
| 765 |
+
"aggregation": "mean",
|
| 766 |
+
"higher_is_better": true,
|
| 767 |
+
"ignore_case": true,
|
| 768 |
+
"ignore_punctuation": true
|
| 769 |
+
}
|
| 770 |
+
],
|
| 771 |
+
"output_type": "generate_until",
|
| 772 |
+
"generation_kwargs": {
|
| 773 |
+
"until": [
|
| 774 |
+
"Question:"
|
| 775 |
+
],
|
| 776 |
+
"max_gen_toks": 32768,
|
| 777 |
+
"do_sample": false,
|
| 778 |
+
"temperature": 1.0,
|
| 779 |
+
"top_p": 0.95,
|
| 780 |
+
"presence_penalty": 1.5
|
| 781 |
+
},
|
| 782 |
+
"repeats": 1,
|
| 783 |
+
"filter_list": [
|
| 784 |
+
{
|
| 785 |
+
"name": "custom-extract",
|
| 786 |
+
"filter": [
|
| 787 |
+
{
|
| 788 |
+
"function": "regex",
|
| 789 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 790 |
+
},
|
| 791 |
+
{
|
| 792 |
+
"function": "take_first"
|
| 793 |
+
}
|
| 794 |
+
]
|
| 795 |
+
}
|
| 796 |
+
],
|
| 797 |
+
"should_decontaminate": false,
|
| 798 |
+
"metadata": {
|
| 799 |
+
"version": 3.1,
|
| 800 |
+
"model": "qwen35-4b-base",
|
| 801 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 802 |
+
"num_concurrent": 16,
|
| 803 |
+
"max_retries": 3,
|
| 804 |
+
"timeout": 1800,
|
| 805 |
+
"tokenized_requests": false,
|
| 806 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml"
|
| 807 |
+
}
|
| 808 |
+
},
|
| 809 |
+
"mmlu_pro_math": {
|
| 810 |
+
"task": "mmlu_pro_math",
|
| 811 |
+
"task_alias": "math",
|
| 812 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 813 |
+
"test_split": "test",
|
| 814 |
+
"fewshot_split": "validation",
|
| 815 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='math')",
|
| 816 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 817 |
+
"doc_to_target": "answer",
|
| 818 |
+
"unsafe_code": false,
|
| 819 |
+
"description": "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 820 |
+
"target_delimiter": " ",
|
| 821 |
+
"fewshot_delimiter": "\n\n",
|
| 822 |
+
"fewshot_config": {
|
| 823 |
+
"sampler": "first_n",
|
| 824 |
+
"split": "validation",
|
| 825 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='math')",
|
| 826 |
+
"fewshot_indices": null,
|
| 827 |
+
"samples": null,
|
| 828 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 829 |
+
"doc_to_choice": null,
|
| 830 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 831 |
+
"gen_prefix": null,
|
| 832 |
+
"fewshot_delimiter": "\n\n",
|
| 833 |
+
"target_delimiter": " "
|
| 834 |
+
},
|
| 835 |
+
"num_fewshot": 5,
|
| 836 |
+
"metric_list": [
|
| 837 |
+
{
|
| 838 |
+
"metric": "exact_match",
|
| 839 |
+
"aggregation": "mean",
|
| 840 |
+
"higher_is_better": true,
|
| 841 |
+
"ignore_case": true,
|
| 842 |
+
"ignore_punctuation": true
|
| 843 |
+
}
|
| 844 |
+
],
|
| 845 |
+
"output_type": "generate_until",
|
| 846 |
+
"generation_kwargs": {
|
| 847 |
+
"until": [
|
| 848 |
+
"Question:"
|
| 849 |
+
],
|
| 850 |
+
"max_gen_toks": 32768,
|
| 851 |
+
"do_sample": false,
|
| 852 |
+
"temperature": 1.0,
|
| 853 |
+
"top_p": 0.95,
|
| 854 |
+
"presence_penalty": 1.5
|
| 855 |
+
},
|
| 856 |
+
"repeats": 1,
|
| 857 |
+
"filter_list": [
|
| 858 |
+
{
|
| 859 |
+
"name": "custom-extract",
|
| 860 |
+
"filter": [
|
| 861 |
+
{
|
| 862 |
+
"function": "regex",
|
| 863 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 864 |
+
},
|
| 865 |
+
{
|
| 866 |
+
"function": "take_first"
|
| 867 |
+
}
|
| 868 |
+
]
|
| 869 |
+
}
|
| 870 |
+
],
|
| 871 |
+
"should_decontaminate": false,
|
| 872 |
+
"metadata": {
|
| 873 |
+
"version": 3.1,
|
| 874 |
+
"model": "qwen35-4b-base",
|
| 875 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 876 |
+
"num_concurrent": 16,
|
| 877 |
+
"max_retries": 3,
|
| 878 |
+
"timeout": 1800,
|
| 879 |
+
"tokenized_requests": false,
|
| 880 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml"
|
| 881 |
+
}
|
| 882 |
+
},
|
| 883 |
+
"mmlu_pro_other": {
|
| 884 |
+
"task": "mmlu_pro_other",
|
| 885 |
+
"task_alias": "other",
|
| 886 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 887 |
+
"test_split": "test",
|
| 888 |
+
"fewshot_split": "validation",
|
| 889 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='other')",
|
| 890 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 891 |
+
"doc_to_target": "answer",
|
| 892 |
+
"unsafe_code": false,
|
| 893 |
+
"description": "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 894 |
+
"target_delimiter": " ",
|
| 895 |
+
"fewshot_delimiter": "\n\n",
|
| 896 |
+
"fewshot_config": {
|
| 897 |
+
"sampler": "first_n",
|
| 898 |
+
"split": "validation",
|
| 899 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='other')",
|
| 900 |
+
"fewshot_indices": null,
|
| 901 |
+
"samples": null,
|
| 902 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 903 |
+
"doc_to_choice": null,
|
| 904 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 905 |
+
"gen_prefix": null,
|
| 906 |
+
"fewshot_delimiter": "\n\n",
|
| 907 |
+
"target_delimiter": " "
|
| 908 |
+
},
|
| 909 |
+
"num_fewshot": 5,
|
| 910 |
+
"metric_list": [
|
| 911 |
+
{
|
| 912 |
+
"metric": "exact_match",
|
| 913 |
+
"aggregation": "mean",
|
| 914 |
+
"higher_is_better": true,
|
| 915 |
+
"ignore_case": true,
|
| 916 |
+
"ignore_punctuation": true
|
| 917 |
+
}
|
| 918 |
+
],
|
| 919 |
+
"output_type": "generate_until",
|
| 920 |
+
"generation_kwargs": {
|
| 921 |
+
"until": [
|
| 922 |
+
"Question:"
|
| 923 |
+
],
|
| 924 |
+
"max_gen_toks": 32768,
|
| 925 |
+
"do_sample": false,
|
| 926 |
+
"temperature": 1.0,
|
| 927 |
+
"top_p": 0.95,
|
| 928 |
+
"presence_penalty": 1.5
|
| 929 |
+
},
|
| 930 |
+
"repeats": 1,
|
| 931 |
+
"filter_list": [
|
| 932 |
+
{
|
| 933 |
+
"name": "custom-extract",
|
| 934 |
+
"filter": [
|
| 935 |
+
{
|
| 936 |
+
"function": "regex",
|
| 937 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 938 |
+
},
|
| 939 |
+
{
|
| 940 |
+
"function": "take_first"
|
| 941 |
+
}
|
| 942 |
+
]
|
| 943 |
+
}
|
| 944 |
+
],
|
| 945 |
+
"should_decontaminate": false,
|
| 946 |
+
"metadata": {
|
| 947 |
+
"version": 3.1,
|
| 948 |
+
"model": "qwen35-4b-base",
|
| 949 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 950 |
+
"num_concurrent": 16,
|
| 951 |
+
"max_retries": 3,
|
| 952 |
+
"timeout": 1800,
|
| 953 |
+
"tokenized_requests": false,
|
| 954 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml"
|
| 955 |
+
}
|
| 956 |
+
},
|
| 957 |
+
"mmlu_pro_philosophy": {
|
| 958 |
+
"task": "mmlu_pro_philosophy",
|
| 959 |
+
"task_alias": "philosophy",
|
| 960 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 961 |
+
"test_split": "test",
|
| 962 |
+
"fewshot_split": "validation",
|
| 963 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='philosophy')",
|
| 964 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 965 |
+
"doc_to_target": "answer",
|
| 966 |
+
"unsafe_code": false,
|
| 967 |
+
"description": "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 968 |
+
"target_delimiter": " ",
|
| 969 |
+
"fewshot_delimiter": "\n\n",
|
| 970 |
+
"fewshot_config": {
|
| 971 |
+
"sampler": "first_n",
|
| 972 |
+
"split": "validation",
|
| 973 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='philosophy')",
|
| 974 |
+
"fewshot_indices": null,
|
| 975 |
+
"samples": null,
|
| 976 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 977 |
+
"doc_to_choice": null,
|
| 978 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 979 |
+
"gen_prefix": null,
|
| 980 |
+
"fewshot_delimiter": "\n\n",
|
| 981 |
+
"target_delimiter": " "
|
| 982 |
+
},
|
| 983 |
+
"num_fewshot": 5,
|
| 984 |
+
"metric_list": [
|
| 985 |
+
{
|
| 986 |
+
"metric": "exact_match",
|
| 987 |
+
"aggregation": "mean",
|
| 988 |
+
"higher_is_better": true,
|
| 989 |
+
"ignore_case": true,
|
| 990 |
+
"ignore_punctuation": true
|
| 991 |
+
}
|
| 992 |
+
],
|
| 993 |
+
"output_type": "generate_until",
|
| 994 |
+
"generation_kwargs": {
|
| 995 |
+
"until": [
|
| 996 |
+
"Question:"
|
| 997 |
+
],
|
| 998 |
+
"max_gen_toks": 32768,
|
| 999 |
+
"do_sample": false,
|
| 1000 |
+
"temperature": 1.0,
|
| 1001 |
+
"top_p": 0.95,
|
| 1002 |
+
"presence_penalty": 1.5
|
| 1003 |
+
},
|
| 1004 |
+
"repeats": 1,
|
| 1005 |
+
"filter_list": [
|
| 1006 |
+
{
|
| 1007 |
+
"name": "custom-extract",
|
| 1008 |
+
"filter": [
|
| 1009 |
+
{
|
| 1010 |
+
"function": "regex",
|
| 1011 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 1012 |
+
},
|
| 1013 |
+
{
|
| 1014 |
+
"function": "take_first"
|
| 1015 |
+
}
|
| 1016 |
+
]
|
| 1017 |
+
}
|
| 1018 |
+
],
|
| 1019 |
+
"should_decontaminate": false,
|
| 1020 |
+
"metadata": {
|
| 1021 |
+
"version": 3.1,
|
| 1022 |
+
"model": "qwen35-4b-base",
|
| 1023 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 1024 |
+
"num_concurrent": 16,
|
| 1025 |
+
"max_retries": 3,
|
| 1026 |
+
"timeout": 1800,
|
| 1027 |
+
"tokenized_requests": false,
|
| 1028 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml"
|
| 1029 |
+
}
|
| 1030 |
+
},
|
| 1031 |
+
"mmlu_pro_physics": {
|
| 1032 |
+
"task": "mmlu_pro_physics",
|
| 1033 |
+
"task_alias": "physics",
|
| 1034 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 1035 |
+
"test_split": "test",
|
| 1036 |
+
"fewshot_split": "validation",
|
| 1037 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='physics')",
|
| 1038 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 1039 |
+
"doc_to_target": "answer",
|
| 1040 |
+
"unsafe_code": false,
|
| 1041 |
+
"description": "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 1042 |
+
"target_delimiter": " ",
|
| 1043 |
+
"fewshot_delimiter": "\n\n",
|
| 1044 |
+
"fewshot_config": {
|
| 1045 |
+
"sampler": "first_n",
|
| 1046 |
+
"split": "validation",
|
| 1047 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='physics')",
|
| 1048 |
+
"fewshot_indices": null,
|
| 1049 |
+
"samples": null,
|
| 1050 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 1051 |
+
"doc_to_choice": null,
|
| 1052 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 1053 |
+
"gen_prefix": null,
|
| 1054 |
+
"fewshot_delimiter": "\n\n",
|
| 1055 |
+
"target_delimiter": " "
|
| 1056 |
+
},
|
| 1057 |
+
"num_fewshot": 5,
|
| 1058 |
+
"metric_list": [
|
| 1059 |
+
{
|
| 1060 |
+
"metric": "exact_match",
|
| 1061 |
+
"aggregation": "mean",
|
| 1062 |
+
"higher_is_better": true,
|
| 1063 |
+
"ignore_case": true,
|
| 1064 |
+
"ignore_punctuation": true
|
| 1065 |
+
}
|
| 1066 |
+
],
|
| 1067 |
+
"output_type": "generate_until",
|
| 1068 |
+
"generation_kwargs": {
|
| 1069 |
+
"until": [
|
| 1070 |
+
"Question:"
|
| 1071 |
+
],
|
| 1072 |
+
"max_gen_toks": 32768,
|
| 1073 |
+
"do_sample": false,
|
| 1074 |
+
"temperature": 1.0,
|
| 1075 |
+
"top_p": 0.95,
|
| 1076 |
+
"presence_penalty": 1.5
|
| 1077 |
+
},
|
| 1078 |
+
"repeats": 1,
|
| 1079 |
+
"filter_list": [
|
| 1080 |
+
{
|
| 1081 |
+
"name": "custom-extract",
|
| 1082 |
+
"filter": [
|
| 1083 |
+
{
|
| 1084 |
+
"function": "regex",
|
| 1085 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 1086 |
+
},
|
| 1087 |
+
{
|
| 1088 |
+
"function": "take_first"
|
| 1089 |
+
}
|
| 1090 |
+
]
|
| 1091 |
+
}
|
| 1092 |
+
],
|
| 1093 |
+
"should_decontaminate": false,
|
| 1094 |
+
"metadata": {
|
| 1095 |
+
"version": 3.1,
|
| 1096 |
+
"model": "qwen35-4b-base",
|
| 1097 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 1098 |
+
"num_concurrent": 16,
|
| 1099 |
+
"max_retries": 3,
|
| 1100 |
+
"timeout": 1800,
|
| 1101 |
+
"tokenized_requests": false,
|
| 1102 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml"
|
| 1103 |
+
}
|
| 1104 |
+
},
|
| 1105 |
+
"mmlu_pro_psychology": {
|
| 1106 |
+
"task": "mmlu_pro_psychology",
|
| 1107 |
+
"task_alias": "psychology",
|
| 1108 |
+
"dataset_path": "TIGER-Lab/MMLU-Pro",
|
| 1109 |
+
"test_split": "test",
|
| 1110 |
+
"fewshot_split": "validation",
|
| 1111 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='psychology')",
|
| 1112 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 1113 |
+
"doc_to_target": "answer",
|
| 1114 |
+
"unsafe_code": false,
|
| 1115 |
+
"description": "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
|
| 1116 |
+
"target_delimiter": " ",
|
| 1117 |
+
"fewshot_delimiter": "\n\n",
|
| 1118 |
+
"fewshot_config": {
|
| 1119 |
+
"sampler": "first_n",
|
| 1120 |
+
"split": "validation",
|
| 1121 |
+
"process_docs": "functools.partial(<function process_docs at 0xe0ddfe7716c0>, subject='psychology')",
|
| 1122 |
+
"fewshot_indices": null,
|
| 1123 |
+
"samples": null,
|
| 1124 |
+
"doc_to_text": "functools.partial(<function format_cot_example at 0xe0ddfe771580>, including_answer=False)",
|
| 1125 |
+
"doc_to_choice": null,
|
| 1126 |
+
"doc_to_target": "functools.partial(<function format_cot_target at 0xe0ddfe771620>, including_answer=True)",
|
| 1127 |
+
"gen_prefix": null,
|
| 1128 |
+
"fewshot_delimiter": "\n\n",
|
| 1129 |
+
"target_delimiter": " "
|
| 1130 |
+
},
|
| 1131 |
+
"num_fewshot": 5,
|
| 1132 |
+
"metric_list": [
|
| 1133 |
+
{
|
| 1134 |
+
"metric": "exact_match",
|
| 1135 |
+
"aggregation": "mean",
|
| 1136 |
+
"higher_is_better": true,
|
| 1137 |
+
"ignore_case": true,
|
| 1138 |
+
"ignore_punctuation": true
|
| 1139 |
+
}
|
| 1140 |
+
],
|
| 1141 |
+
"output_type": "generate_until",
|
| 1142 |
+
"generation_kwargs": {
|
| 1143 |
+
"until": [
|
| 1144 |
+
"Question:"
|
| 1145 |
+
],
|
| 1146 |
+
"max_gen_toks": 32768,
|
| 1147 |
+
"do_sample": false,
|
| 1148 |
+
"temperature": 1.0,
|
| 1149 |
+
"top_p": 0.95,
|
| 1150 |
+
"presence_penalty": 1.5
|
| 1151 |
+
},
|
| 1152 |
+
"repeats": 1,
|
| 1153 |
+
"filter_list": [
|
| 1154 |
+
{
|
| 1155 |
+
"name": "custom-extract",
|
| 1156 |
+
"filter": [
|
| 1157 |
+
{
|
| 1158 |
+
"function": "regex",
|
| 1159 |
+
"regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
|
| 1160 |
+
},
|
| 1161 |
+
{
|
| 1162 |
+
"function": "take_first"
|
| 1163 |
+
}
|
| 1164 |
+
]
|
| 1165 |
+
}
|
| 1166 |
+
],
|
| 1167 |
+
"should_decontaminate": false,
|
| 1168 |
+
"metadata": {
|
| 1169 |
+
"version": 3.1,
|
| 1170 |
+
"model": "qwen35-4b-base",
|
| 1171 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 1172 |
+
"num_concurrent": 16,
|
| 1173 |
+
"max_retries": 3,
|
| 1174 |
+
"timeout": 1800,
|
| 1175 |
+
"tokenized_requests": false,
|
| 1176 |
+
"config_source": "/home/unsloth/.local/lib/python3.12/site-packages/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml"
|
| 1177 |
+
}
|
| 1178 |
+
}
|
| 1179 |
+
},
|
| 1180 |
+
"versions": {
|
| 1181 |
+
"mmlu_pro": "2.0",
|
| 1182 |
+
"mmlu_pro_biology": 3.1,
|
| 1183 |
+
"mmlu_pro_business": 3.1,
|
| 1184 |
+
"mmlu_pro_chemistry": 3.1,
|
| 1185 |
+
"mmlu_pro_computer_science": 3.1,
|
| 1186 |
+
"mmlu_pro_economics": 3.1,
|
| 1187 |
+
"mmlu_pro_engineering": 3.1,
|
| 1188 |
+
"mmlu_pro_health": 3.1,
|
| 1189 |
+
"mmlu_pro_history": 3.1,
|
| 1190 |
+
"mmlu_pro_law": 3.1,
|
| 1191 |
+
"mmlu_pro_math": 3.1,
|
| 1192 |
+
"mmlu_pro_other": 3.1,
|
| 1193 |
+
"mmlu_pro_philosophy": 3.1,
|
| 1194 |
+
"mmlu_pro_physics": 3.1,
|
| 1195 |
+
"mmlu_pro_psychology": 3.1
|
| 1196 |
+
},
|
| 1197 |
+
"n-shot": {
|
| 1198 |
+
"mmlu_pro": 5,
|
| 1199 |
+
"mmlu_pro_biology": 5,
|
| 1200 |
+
"mmlu_pro_business": 5,
|
| 1201 |
+
"mmlu_pro_chemistry": 5,
|
| 1202 |
+
"mmlu_pro_computer_science": 5,
|
| 1203 |
+
"mmlu_pro_economics": 5,
|
| 1204 |
+
"mmlu_pro_engineering": 5,
|
| 1205 |
+
"mmlu_pro_health": 5,
|
| 1206 |
+
"mmlu_pro_history": 5,
|
| 1207 |
+
"mmlu_pro_law": 5,
|
| 1208 |
+
"mmlu_pro_math": 5,
|
| 1209 |
+
"mmlu_pro_other": 5,
|
| 1210 |
+
"mmlu_pro_philosophy": 5,
|
| 1211 |
+
"mmlu_pro_physics": 5,
|
| 1212 |
+
"mmlu_pro_psychology": 5
|
| 1213 |
+
},
|
| 1214 |
+
"higher_is_better": {
|
| 1215 |
+
"mmlu_pro": {
|
| 1216 |
+
"exact_match": true
|
| 1217 |
+
},
|
| 1218 |
+
"mmlu_pro_biology": {
|
| 1219 |
+
"exact_match": true
|
| 1220 |
+
},
|
| 1221 |
+
"mmlu_pro_business": {
|
| 1222 |
+
"exact_match": true
|
| 1223 |
+
},
|
| 1224 |
+
"mmlu_pro_chemistry": {
|
| 1225 |
+
"exact_match": true
|
| 1226 |
+
},
|
| 1227 |
+
"mmlu_pro_computer_science": {
|
| 1228 |
+
"exact_match": true
|
| 1229 |
+
},
|
| 1230 |
+
"mmlu_pro_economics": {
|
| 1231 |
+
"exact_match": true
|
| 1232 |
+
},
|
| 1233 |
+
"mmlu_pro_engineering": {
|
| 1234 |
+
"exact_match": true
|
| 1235 |
+
},
|
| 1236 |
+
"mmlu_pro_health": {
|
| 1237 |
+
"exact_match": true
|
| 1238 |
+
},
|
| 1239 |
+
"mmlu_pro_history": {
|
| 1240 |
+
"exact_match": true
|
| 1241 |
+
},
|
| 1242 |
+
"mmlu_pro_law": {
|
| 1243 |
+
"exact_match": true
|
| 1244 |
+
},
|
| 1245 |
+
"mmlu_pro_math": {
|
| 1246 |
+
"exact_match": true
|
| 1247 |
+
},
|
| 1248 |
+
"mmlu_pro_other": {
|
| 1249 |
+
"exact_match": true
|
| 1250 |
+
},
|
| 1251 |
+
"mmlu_pro_philosophy": {
|
| 1252 |
+
"exact_match": true
|
| 1253 |
+
},
|
| 1254 |
+
"mmlu_pro_physics": {
|
| 1255 |
+
"exact_match": true
|
| 1256 |
+
},
|
| 1257 |
+
"mmlu_pro_psychology": {
|
| 1258 |
+
"exact_match": true
|
| 1259 |
+
}
|
| 1260 |
+
},
|
| 1261 |
+
"n-samples": {
|
| 1262 |
+
"mmlu_pro_biology": {
|
| 1263 |
+
"original": 717,
|
| 1264 |
+
"effective": 25
|
| 1265 |
+
},
|
| 1266 |
+
"mmlu_pro_business": {
|
| 1267 |
+
"original": 789,
|
| 1268 |
+
"effective": 25
|
| 1269 |
+
},
|
| 1270 |
+
"mmlu_pro_chemistry": {
|
| 1271 |
+
"original": 1132,
|
| 1272 |
+
"effective": 25
|
| 1273 |
+
},
|
| 1274 |
+
"mmlu_pro_computer_science": {
|
| 1275 |
+
"original": 410,
|
| 1276 |
+
"effective": 25
|
| 1277 |
+
},
|
| 1278 |
+
"mmlu_pro_economics": {
|
| 1279 |
+
"original": 844,
|
| 1280 |
+
"effective": 25
|
| 1281 |
+
},
|
| 1282 |
+
"mmlu_pro_engineering": {
|
| 1283 |
+
"original": 969,
|
| 1284 |
+
"effective": 25
|
| 1285 |
+
},
|
| 1286 |
+
"mmlu_pro_health": {
|
| 1287 |
+
"original": 818,
|
| 1288 |
+
"effective": 25
|
| 1289 |
+
},
|
| 1290 |
+
"mmlu_pro_history": {
|
| 1291 |
+
"original": 381,
|
| 1292 |
+
"effective": 25
|
| 1293 |
+
},
|
| 1294 |
+
"mmlu_pro_law": {
|
| 1295 |
+
"original": 1101,
|
| 1296 |
+
"effective": 25
|
| 1297 |
+
},
|
| 1298 |
+
"mmlu_pro_math": {
|
| 1299 |
+
"original": 1351,
|
| 1300 |
+
"effective": 25
|
| 1301 |
+
},
|
| 1302 |
+
"mmlu_pro_other": {
|
| 1303 |
+
"original": 924,
|
| 1304 |
+
"effective": 25
|
| 1305 |
+
},
|
| 1306 |
+
"mmlu_pro_philosophy": {
|
| 1307 |
+
"original": 499,
|
| 1308 |
+
"effective": 25
|
| 1309 |
+
},
|
| 1310 |
+
"mmlu_pro_physics": {
|
| 1311 |
+
"original": 1299,
|
| 1312 |
+
"effective": 25
|
| 1313 |
+
},
|
| 1314 |
+
"mmlu_pro_psychology": {
|
| 1315 |
+
"original": 798,
|
| 1316 |
+
"effective": 25
|
| 1317 |
+
}
|
| 1318 |
+
},
|
| 1319 |
+
"config": {
|
| 1320 |
+
"model": "local-chat-completions",
|
| 1321 |
+
"model_args": {
|
| 1322 |
+
"model": "qwen35-4b-base",
|
| 1323 |
+
"base_url": "http://172.17.0.1:8082/v1/chat/completions",
|
| 1324 |
+
"num_concurrent": 16,
|
| 1325 |
+
"max_retries": 3,
|
| 1326 |
+
"timeout": 1800,
|
| 1327 |
+
"tokenized_requests": false
|
| 1328 |
+
},
|
| 1329 |
+
"batch_size": 1,
|
| 1330 |
+
"batch_sizes": [],
|
| 1331 |
+
"device": "cuda:0",
|
| 1332 |
+
"use_cache": null,
|
| 1333 |
+
"limit": 25.0,
|
| 1334 |
+
"bootstrap_iters": 100000,
|
| 1335 |
+
"gen_kwargs": {
|
| 1336 |
+
"temperature": 1.0,
|
| 1337 |
+
"top_p": 0.95,
|
| 1338 |
+
"presence_penalty": 1.5,
|
| 1339 |
+
"max_gen_toks": 32768
|
| 1340 |
+
},
|
| 1341 |
+
"random_seed": 42,
|
| 1342 |
+
"numpy_seed": 42,
|
| 1343 |
+
"torch_seed": 42,
|
| 1344 |
+
"fewshot_seed": 42
|
| 1345 |
+
},
|
| 1346 |
+
"git_hash": null,
|
| 1347 |
+
"date": 1784206951.8514302,
|
| 1348 |
+
"pretty_env_info": "PyTorch version: 2.10.0a0+b558c986e8.nv25.11\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.3 LTS (aarch64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: version 3.31.6\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Aug 14 2025, 17:47:21) [GCC 13.3.0] (64-bit runtime)\nPython platform: Linux-6.17.0-1014-nvidia-aarch64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 13.0.88\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GB10\nNvidia driver version: 580.142\ncuDNN version: Probably one of the following:\n/usr/lib/aarch64-linux-gnu/libcudnn.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_adv.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_cnn.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_engines_precompiled.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_graph.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_heuristic.so.9.15.0\n/usr/lib/aarch64-linux-gnu/libcudnn_ops.so.9.15.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: aarch64\nCPU op-mode(s): 64-bit\nByte Order: Little Endian\nCPU(s): 20\nOn-line CPU(s) list: 0-19\nVendor ID: ARM\nModel name: Cortex-X925\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 10\nSocket(s): 1\nStepping: r0p1\nFrequency boost: disabled\nCPU(s) scaling MHz: 100%\nCPU max MHz: 3900.0000\nCPU min MHz: 1378.0000\nBogoMIPS: 2000.00\nFlags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm sb paca pacg dcpodp sve2 sveaes svepmull svebitperm svesha3 svesm4 flagm2 frint svei8mm svebf16 i8mm bf16 dgh bti ecv afp wfxt\nModel name: Cortex-A725\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 10\nSocket(s): 1\nStepping: r0p1\nCPU(s) scaling MHz: 100%\nCPU max MHz: 2808.0000\nCPU min MHz: 338.0000\nBogoMIPS: 2000.00\nFlags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm sb paca pacg dcpodp sve2 sveaes svepmull svebitperm svesha3 svesm4 flagm2 frint svei8mm svebf16 i8mm bf16 dgh bti ecv afp wfxt\nL1d cache: 1.3 MiB (20 instances)\nL1i cache: 1.3 MiB (20 instances)\nL2 cache: 25 MiB (20 instances)\nL3 cache: 24 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-19\nVulnerability Gather data sampling: Not affected\nVulnerability Ghostwrite: Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Old microcode: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; __user pointer sanitization\nVulnerability Spectre v2: Mitigation; CSV2, BHB\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Not affected\nVulnerability Vmscape: Not affected\n\nVersions of relevant libraries:\n[pip3] executorch==1.1.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==2.4.1\n[pip3] nvidia-cudnn-frontend==1.15.0\n[pip3] nvtx==0.2.13\n[pip3] onnx==1.18.0\n[pip3] onnx-ir==0.1.12\n[pip3] onnxscript==0.5.6\n[pip3] optree==0.17.0\n[pip3] pytorch_tokenizers==1.1.0\n[pip3] pytorch-triton==3.5.0+gitde3506d2\n[pip3] torch==2.10.0a0+b558c986e8.nv25.11\n[pip3] torch_c_dlpack_ext==0.1.5\n[pip3] torch-stoi==0.2.3\n[pip3] torch_tensorrt==2.10.0a0\n[pip3] torchao==0.14.0+git\n[pip3] torchaudio==2.9.1+a224ab2\n[pip3] torchcodec==0.11.0a0\n[pip3] torchdata==0.11.0\n[pip3] torchprofile==0.0.4\n[pip3] torchtitan==0.1.0\n[pip3] torchvision==0.25.0a0+7a13ad0f\n[pip3] triton==3.5.1\n[pip3] triton_kernels==1.0.0+nv25.11\n[conda] Could not collect",
|
| 1349 |
+
"transformers_version": "5.3.0",
|
| 1350 |
+
"lm_eval_version": "0.4.12",
|
| 1351 |
+
"upper_git_hash": null,
|
| 1352 |
+
"task_hashes": {
|
| 1353 |
+
"mmlu_pro_biology": "d299003b29307be899aac2b36c294706ead0d1fa70da8055c16c64948e47bb37",
|
| 1354 |
+
"mmlu_pro_business": "51d20700c67274e0f13bb21fab446f0d8c9a5b4e99bde489b8661aabe0772d73",
|
| 1355 |
+
"mmlu_pro_chemistry": "cbbce9655c0e134685346faa69d556ef652777e835bbb9f5af1552515a33c91b",
|
| 1356 |
+
"mmlu_pro_computer_science": "c7ebe611e4d3037b6edeb2f5d5d8b3b9af40184a0921909fa56a0e5899931329",
|
| 1357 |
+
"mmlu_pro_economics": "27da2bf1ab32c605b21b413ba8d4ff2e31eecc5c549c20f52954f7686e877ab2",
|
| 1358 |
+
"mmlu_pro_engineering": "6c2436a6a4fefe6d8fcf44dd3071974821b391bcd009218bfde8d8a89197e948",
|
| 1359 |
+
"mmlu_pro_health": "73976f6cc6c06e372b9ea96864e1aa188663d87188b902b91b94f8f0b7daea79",
|
| 1360 |
+
"mmlu_pro_history": "32c13befde4ed188ef3ba489c6430a7e64d77220ef27eba63fb8536455665a22",
|
| 1361 |
+
"mmlu_pro_law": "03139fe875893727cc5046cd21a13334eaa9b65f45a7521894c01baf8d9620d9",
|
| 1362 |
+
"mmlu_pro_math": "ac2d2e488bacf3ad772582ff958d5f7d546f4e6e0166d451b8814f4d8d96a118",
|
| 1363 |
+
"mmlu_pro_other": "0cbddd8cff9dcd2c9dd0586bc5912e180bff1672d08d7ec9d685657e58329370",
|
| 1364 |
+
"mmlu_pro_philosophy": "27d010aa4c9ea2f5b9bdfc9012616d202a061a7e6cf9cba39a44d7376e17628b",
|
| 1365 |
+
"mmlu_pro_physics": "7b280effbf525c36aa8262ecaa7f4edd7830d7d9715c21bd465ab4af695234be",
|
| 1366 |
+
"mmlu_pro_psychology": "e1e218e9af86c6c467f2d64e6c55df06e2db06b077b72c682cb6b709591f88c0"
|
| 1367 |
+
},
|
| 1368 |
+
"model_source": "local-chat-completions",
|
| 1369 |
+
"model_name": "qwen35-4b-base",
|
| 1370 |
+
"model_name_sanitized": "qwen35-4b-base",
|
| 1371 |
+
"system_instruction": null,
|
| 1372 |
+
"system_instruction_sha": null,
|
| 1373 |
+
"fewshot_as_multiturn": true,
|
| 1374 |
+
"chat_template": "",
|
| 1375 |
+
"chat_template_sha": null,
|
| 1376 |
+
"total_evaluation_time_seconds": "2841.5148988645524"
|
| 1377 |
+
}
|