{ "release_step": 87776, "methodology": { "temperature": 0, "human_eval_stop": "same top-level stop rules for both models", "token_limit": "768 total tokens using each model tokenizer; EAR task includes forced prefix", "executor": "Docker isolated, network disabled", "human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef", "mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9", "reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor", "limitations": "Models use different tokenizers and interfaces; the matched 20-task comparison is too small to establish a general ranking; HumanEval prompt forced as prefix and counted within output limit; MBPP is a random 30-task sample with one visible test; training-set contamination not excluded.", "market_eval": "research/bench_humaneval_market.py" }, "earai": { "humaneval_full": { "tasks": 164, "passed": 0, "syntax_valid": 123, "prefix_exceeded": 17 }, "mbpp_sanitized_sample": { "tasks": 30, "passed": 0, "syntax_valid": 0, "prefix_exceeded": 0 }, "humaneval_matched_subset": { "tasks": 20, "passed": 0, "syntax_valid": 15, "prefix_exceeded": 3 }, "mbpp_matched_subset": { "tasks": 10, "passed": 0, "syntax_valid": 0, "prefix_exceeded": 0 } }, "tinystarcoder_py": { "parameters": 164144128, "humaneval_matched_subset": { "tasks": 20, "passed": 1, "syntax_valid": 19, "prefix_exceeded": 0 }, "mbpp_matched_subset": { "tasks": 10, "passed": 0, "syntax_valid": 0, "prefix_exceeded": 0 } }, "qwen25_0_5b": { "parameters": 494032768, "humaneval_matched_subset": { "tasks": 20, "passed": 7, "syntax_valid": 19, "prefix_exceeded": 0 }, "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation" }, "qwen25_coder_0_5b": { "parameters": 494032768, "humaneval_matched_subset": { "tasks": 20, "passed": 3, "syntax_valid": 19, "prefix_exceeded": 0 }, "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation" } }