File size: 1,139 Bytes
a31a731 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Held-out eval used BEFORE every checkpoint save."""
from __future__ import annotations
from typing import Any, Callable
from deepagents_rl.rewards.unit_test import compute_reward, grade_completion
def eval_on_problems(
generate_fn: Callable[[str], str],
problems: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""
generate_fn(prompt) -> completion text.
Returns per-problem grade/reward rows for Langfuse + W&B.
"""
rows: list[dict[str, Any]] = []
for p in problems:
prompt = p["prompt"]
completion = generate_fn(prompt)
grade = grade_completion(completion, list(p["hidden_tests"]))
reward = compute_reward(grade, format_ok=True, ran_execute=False)
rows.append(
{
"task_id": p["task_id"],
"prompt": prompt,
"completion": completion,
"pass_frac": reward["pass_frac"],
"reward_total": reward["reward_total"],
"passed": grade["passed"],
"total": grade["total"],
}
)
return rows
|