LFM2.5-2.6B-opencode-RL / eval_results.json
AdithyaSK's picture
AdithyaSK HF Staff
Release step 1000 with tokenizer, evaluation results and model card
99b036c verified
Raw History Blame Contribute Delete
1.81 kB
{
"metric": "pass@1",
"complete": true,
"graded_cells": 1000,
"average_pass_at_1": 0.523,
"harnesses": {
"opencode": {
"evaluations": 250,
"pass_at_1": 0.58,
"difficulty": {
"easy": {
"evaluations": 33,
"pass_at_1": 0.9090909090909091
},
"medium": {
"evaluations": 118,
"pass_at_1": 0.6101694915254238
},
"hard": {
"evaluations": 99,
"pass_at_1": 0.43434343434343436
}
}
},
"claude-code": {
"evaluations": 250,
"pass_at_1": 0.42,
"difficulty": {
"easy": {
"evaluations": 33,
"pass_at_1": 0.6060606060606061
},
"medium": {
"evaluations": 118,
"pass_at_1": 0.4576271186440678
},
"hard": {
"evaluations": 99,
"pass_at_1": 0.31313131313131315
}
}
},
"codex": {
"evaluations": 250,
"pass_at_1": 0.432,
"difficulty": {
"easy": {
"evaluations": 33,
"pass_at_1": 0.6666666666666666
},
"medium": {
"evaluations": 118,
"pass_at_1": 0.4915254237288136
},
"hard": {
"evaluations": 99,
"pass_at_1": 0.2828282828282828
}
}
},
"mini-swe-agent": {
"evaluations": 250,
"pass_at_1": 0.66,
"difficulty": {
"easy": {
"evaluations": 33,
"pass_at_1": 0.8787878787878788
},
"medium": {
"evaluations": 118,
"pass_at_1": 0.6864406779661016
},
"hard": {
"evaluations": 99,
"pass_at_1": 0.5555555555555556
}
}
}
},
"tito_pass": true,
"comparison_ready": true
}