File size: 4,041 Bytes
dafe8d2 079e13c dafe8d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Spawn or collect the deployed beam-dory-code app. No candidate execution locally.
MODAL_PROFILE=aside modal deploy modal_app/code_eval.py::APP --env main
MODAL_PROFILE=aside python3 scripts/spawn_code_eval.py spawn --run smoke-20261030 --configs base --limit 2 --runtime-s 360
MODAL_PROFILE=aside python3 scripts/spawn_code_eval.py spawn --run full-20261030
MODAL_PROFILE=aside python3 scripts/spawn_code_eval.py collect --run full-20261030
"""
import argparse
import json
from pathlib import Path
import modal
CONFIGS = ["base", "rE-t050-default", "rE-t050-low", "rE-t050-high", "rA-t030-default",
"rC-t050-default", "rC-t050-low", "rC-t050-high"]
LOCAL = Path(__file__).resolve().parents[1] / "data/eval/code"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["spawn", "collect", "status"])
parser.add_argument("--run", required=True)
parser.add_argument("--configs", default=",".join(CONFIGS))
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--runtime-s", type=int, default=2700)
args = parser.parse_args()
configs = args.configs.split(",")
assert set(configs) <= set(CONFIGS)
path = LOCAL / args.run
path.mkdir(parents=True, exist_ok=True)
calls_path = path / "calls.json"
if args.action == "spawn":
if calls_path.exists():
raise FileExistsError(calls_path)
fn = modal.Function.from_name("beam-dory-code", "evaluate", environment_name="main")
calls = {}
for config in configs:
call = fn.spawn(config, args.run, args.limit, args.runtime_s)
calls[config] = call.object_id
calls_path.write_text(json.dumps(calls, indent=2) + "\n")
print(config, call.object_id, flush=True)
return
calls = json.loads(calls_path.read_text())
volume = modal.Volume.from_name("beam-dory-data", environment_name="main")
for config, call_id in calls.items():
if config not in configs:
continue
try:
result = modal.FunctionCall.from_id(call_id).get(timeout=0)
print(config, "generation:", result["state"], flush=True)
except (TimeoutError, modal.exception.TimeoutError):
print(config, "generation: running", flush=True)
remote = f"eval/code/{args.run}/{config}"
dest = path / config
dest.mkdir(exist_ok=True)
for name in ("manifest.json", "progress.json", "summary.json", "results.jsonl", "prompts.jsonl"):
try:
contents = b"".join(volume.read_file(f"{remote}/{name}"))
(dest / name).write_bytes(contents)
if name in ("progress.json", "summary.json"):
print(config, name, contents.decode().strip(), flush=True)
except FileNotFoundError:
pass
manifest_file = dest / "manifest.json"
if manifest_file.exists():
manifest = json.loads(manifest_file.read_text())
grade_id = manifest.get("grade_call_id")
if grade_id:
try:
graded = modal.FunctionCall.from_id(grade_id).get(timeout=0)
print(config, "grading: done", graded["benches"], flush=True)
except (TimeoutError, modal.exception.TimeoutError):
print(config, "grading: running", flush=True)
# Raw generations contain model-written code as inert JSON strings.
# Collection deliberately does not import, compile, or execute any of it.
if args.action == "collect":
for name in ("generations.jsonl", "sanitized.jsonl"):
try:
with (dest / name).open("wb") as handle:
for chunk in volume.read_file(f"{remote}/{name}"):
handle.write(chunk)
except FileNotFoundError:
(dest / name).unlink(missing_ok=True)
if __name__ == "__main__":
main()
|