| """Accuracy under thinking budgets (2k/4k/8k/16k tokens) with budget forcing, reusing the 64k eval samples. |
| |
| MODAL_PROFILE=aside modal deploy modal_app/eval_dory.py::APP |
| MODAL_PROFILE=aside python3 scripts/spawn_budget.py |
| |
| Uploads each result file to the beam-dory-data Volume (eval/budget_inputs/) and spawns one budget_force call per |
| configuration. Results land in eval/results/<model>/budget-*.jsonl. |
| """ |
| import json |
| import subprocess |
| from pathlib import Path |
|
|
| import modal |
|
|
| RES = Path(__file__).resolve().parent.parent / "data" / "eval" / "results" |
| CAPS = [2048, 4096, 8192, 16384] |
| |
| CONFIGS = [ |
| ("base_loops2-64k", "base", "", False), |
| ("base_loops2eff", "base", "", True), |
| ("rE-t050_loops2-low-64k", "merges/rE-t050", "low", False), |
| ("rE-t050_loops2-default-64k", "merges/rE-t050", "", False), |
| ("rE-t050_loops2-high-64k", "merges/rE-t050", "high", False), |
| ("mix030_loops2-64k", "merges/rA-t030", "", False), |
| ] |
|
|
| fn = modal.Function.from_name("beam-dory-eval", "budget_force", environment_name="main") |
| for tag, model, effort, eff in CONFIGS: |
| dest = f"eval/budget_inputs/{tag}.jsonl" |
| subprocess.run(["modal", "volume", "put", "-f", "beam-dory-data", str(RES / f"{tag}.jsonl"), dest], |
| check=True, capture_output=True) |
| call = fn.spawn(model, 2, dest, CAPS, effort, eff) |
| print(json.dumps({"tag": tag, "model": model, "effort": effort, "efficient": eff, "call": call.object_id}), flush=True) |
|
|
