EARAI0.1 / benchmark_outcomes.jsonl
EARAI's picture
Release EARAI 0.1 research checkpoint
2100fa6 verified
Raw History Blame Contribute Delete
30.2 kB
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/0", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/1", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/2", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/3", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/4", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/5", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/6", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/7", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/8", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/9", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/10", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/11", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/12", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/13", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/14", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/15", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/16", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/17", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/18", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/19", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/20", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/21", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/22", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/23", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/24", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/25", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/26", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/27", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/28", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/29", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/30", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/31", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/32", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/33", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/34", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/35", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/36", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/37", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/38", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/39", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/40", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/41", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/42", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/43", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/44", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/45", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/46", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/47", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/48", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/49", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/50", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/51", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/52", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/53", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/54", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/55", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/56", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/57", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/58", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/59", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/60", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/61", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/62", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/63", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/64", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/65", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/66", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/67", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/68", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/69", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/70", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/71", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/72", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/73", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/74", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/75", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/76", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/77", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/78", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/79", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/80", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/81", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/82", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/83", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/84", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/85", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/86", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/87", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/88", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/89", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/90", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/91", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/92", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/93", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/94", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/95", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/96", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/97", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/98", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/99", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/100", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/101", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/102", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/103", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/104", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/105", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/106", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/107", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/108", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/109", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/110", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/111", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/112", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/113", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/114", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/115", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/116", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/117", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/118", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/119", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/120", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/121", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/122", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/123", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/124", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/125", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/126", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/127", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/128", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/129", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/130", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/131", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/132", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/133", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/134", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/135", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/136", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/137", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/138", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/139", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/140", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/141", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/142", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/143", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/144", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/145", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/146", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/147", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/148", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/149", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/150", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/151", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/152", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/153", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/154", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/155", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/156", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/157", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/158", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/159", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/160", "passed": false, "syntax_valid": false, "unsupported": true}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/161", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/162", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "HumanEval", "task_id": "HumanEval/163", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/30", "passed": true, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/81", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/128", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/131", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/26", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/57", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/153", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/159", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/142", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/107", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/146", "passed": false, "syntax_valid": false, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/140", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/125", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/150", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/112", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/61", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/0", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/157", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/20", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "TinyStarCoderPy", "benchmark": "HumanEval-matched-sample", "task_id": "HumanEval/28", "passed": false, "syntax_valid": true, "unsupported": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 106, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 293, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 473, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 478, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 629, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 98, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 235, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 602, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 615, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 578, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 427, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 782, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 587, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 573, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 752, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 778, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 773, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 464, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 763, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 776, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 594, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 437, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 246, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 3, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 610, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 87, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 102, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 274, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 801, "passed": false, "syntax_valid": false}
{"model": "EARAI0.1", "benchmark": "MBPP-sanitized-sample", "task_id": 96, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 578, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 102, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 478, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 752, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 106, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 437, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 778, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 610, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 427, "passed": false, "syntax_valid": false}
{"model": "TinyStarCoderPy", "benchmark": "MBPP-matched-sample", "task_id": 235, "passed": false, "syntax_valid": false}