File size: 1,527 Bytes
dafe8d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Accuracy under thinking budgets (2k/4k/8k/16k tokens) with budget forcing, reusing the 64k eval samples.
MODAL_PROFILE=aside modal deploy modal_app/eval_dory.py::APP
MODAL_PROFILE=aside python3 scripts/spawn_budget.py
Uploads each result file to the beam-dory-data Volume (eval/budget_inputs/) and spawns one budget_force call per
configuration. Results land in eval/results/<model>/budget-*.jsonl.
"""
import json
import subprocess
from pathlib import Path
import modal
RES = Path(__file__).resolve().parent.parent / "data" / "eval" / "results"
CAPS = [2048, 4096, 8192, 16384]
# (local result file, model on the Volume, reasoning_effort, medium_effort)
CONFIGS = [
("base_loops2-64k", "base", "", False),
("base_loops2eff", "base", "", True),
("rE-t050_loops2-low-64k", "merges/rE-t050", "low", False),
("rE-t050_loops2-default-64k", "merges/rE-t050", "", False),
("rE-t050_loops2-high-64k", "merges/rE-t050", "high", False),
("mix030_loops2-64k", "merges/rA-t030", "", False),
]
fn = modal.Function.from_name("beam-dory-eval", "budget_force", environment_name="main")
for tag, model, effort, eff in CONFIGS:
dest = f"eval/budget_inputs/{tag}.jsonl"
subprocess.run(["modal", "volume", "put", "-f", "beam-dory-data", str(RES / f"{tag}.jsonl"), dest],
check=True, capture_output=True)
call = fn.spawn(model, 2, dest, CAPS, effort, eff)
print(json.dumps({"tag": tag, "model": model, "effort": effort, "efficient": eff, "call": call.object_id}), flush=True)
|