| """Held-out eval used BEFORE every checkpoint save.""" | |
| from __future__ import annotations | |
| from typing import Any, Callable | |
| from deepagents_rl.rewards.unit_test import compute_reward, grade_completion | |
| def eval_on_problems( | |
| generate_fn: Callable[[str], str], | |
| problems: list[dict[str, Any]], | |
| ) -> list[dict[str, Any]]: | |
| """ | |
| generate_fn(prompt) -> completion text. | |
| Returns per-problem grade/reward rows for Langfuse + W&B. | |
| """ | |
| rows: list[dict[str, Any]] = [] | |
| for p in problems: | |
| prompt = p["prompt"] | |
| completion = generate_fn(prompt) | |
| grade = grade_completion(completion, list(p["hidden_tests"])) | |
| reward = compute_reward(grade, format_ok=True, ran_execute=False) | |
| rows.append( | |
| { | |
| "task_id": p["task_id"], | |
| "prompt": prompt, | |
| "completion": completion, | |
| "pass_frac": reward["pass_frac"], | |
| "reward_total": reward["reward_total"], | |
| "passed": grade["passed"], | |
| "total": grade["total"], | |
| } | |
| ) | |
| return rows | |