| """ |
| This script aims at quickly initialize a sketch for HumanEvalPlus. It's not going to be |
| perfect, but we will either manually or automatically fix/complete it later. |
| + CHANGE 1: Adds "contract", "base_input", "atol" in addition to HumanEval. |
| """ |
|
|
| import json |
| import os |
| import pathlib |
| from importlib import import_module |
| from inspect import getsource |
| from typing import Tuple |
|
|
| from tempdir import TempDir |
|
|
| from evalplus.data.humaneval import get_human_eval |
|
|
| HUMANEVAL_PLUS_PATH = ( |
| pathlib.Path(__file__).parent.parent.parent / "HumanEvalPlus.jsonl" |
| ) |
|
|
|
|
| def _ret(entry_point) -> str: |
| """This is a hacky function to return some garbages so that we can |
| successfully run the function . |
| """ |
| if entry_point == "sort_third" or entry_point == "sort_even": |
| return [1, 2, 3] |
| elif entry_point == "bf": |
| return () |
| return "1" |
|
|
|
|
| def instrument_inputs(entry_point, prompt, test) -> str: |
| globals()["_inputs"] = [] |
| fn_text = f"""{prompt.split(f"def {entry_point}")[0]} |
| |
| def {entry_point}(*args): |
| _inputs.append(args) |
| return {_ret(entry_point)} |
| """ |
| exec(fn_text, globals()) |
| exec(test.replace("assert ", ""), globals()) |
| exec(f"check({entry_point})", globals()) |
| exec(fn_text, globals()) |
| return globals()["_inputs"] |
|
|
|
|
| def get_contract_and_ref(task_id: int, entry_point) -> Tuple[str, str]: |
| mod = import_module(f"groundtruth.humaneval.{str(task_id).zfill(3)}_{entry_point}") |
| fn = getattr(mod, entry_point) |
|
|
| doc = fn.__doc__ |
| if task_id == 51: |
| doc = doc.replace("bcdf\nghjklm", r"bcdf\nghjklm").replace( |
| "abcdef\nghijklm", r"abcdef\nghijklm" |
| ) |
|
|
| code = ( |
| getsource(fn).replace(doc, "").replace("''''''", '""""""').split('""""""\n')[-1] |
| ) |
|
|
| assert code, f"Something wrong with {task_id}!" |
| assert code[:3] != "def", f"Something wrong with the {task_id}!" |
|
|
| |
| contract = "" |
| impl = "" |
|
|
| reading_contract = True |
| for line in code.strip("\n").split("\n"): |
| if reading_contract and "$_CONTRACT_$" in line: |
| contract += line + "\n" |
| else: |
| reading_contract = False |
| impl += line + "\n" |
|
|
| if contract: |
| contract = "\n" + contract |
|
|
| return contract, "\n" + impl + "\n" |
|
|
|
|
| def get_atol(task_id: int) -> float: |
| if task_id == 2 or task_id == 4: |
| return 1e-6 |
| elif task_id == 32: |
| return 1e-4 |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| assert not HUMANEVAL_PLUS_PATH.exists(), f"{HUMANEVAL_PLUS_PATH} already exists!" |
|
|
| human_eval = get_human_eval() |
| with TempDir() as temp_dir: |
| tmp_file = os.path.join(temp_dir, HUMANEVAL_PLUS_PATH) |
| with open(tmp_file, "w") as writer: |
| for task in human_eval: |
| task_id = int(task["task_id"].split("/")[-1]) |
| task["contract"], task["canonical_solution"] = get_contract_and_ref( |
| task_id, task["entry_point"] |
| ) |
| task["base_input"] = instrument_inputs( |
| task["entry_point"], task["prompt"], task["test"] |
| ) |
| task["atol"] = get_atol(task_id) |
| task["task_id"] = task["task_id"] |
|
|
| writer.write(json.dumps(task) + "\n") |
| |
| os.rename(tmp_file, HUMANEVAL_PLUS_PATH) |
|
|
