"""Held-out eval used BEFORE every checkpoint save."""
from __future__ import annotations

from typing import Any, Callable

from deepagents_rl.rewards.unit_test import compute_reward, grade_completion


def eval_on_problems(
    generate_fn: Callable[[str], str],
    problems: list[dict[str, Any]],
) -> list[dict[str, Any]]:
    """
    generate_fn(prompt) -> completion text.
    Returns per-problem grade/reward rows for Langfuse + W&B.
    """
    rows: list[dict[str, Any]] = []
    for p in problems:
        prompt = p["prompt"]
        completion = generate_fn(prompt)
        grade = grade_completion(completion, list(p["hidden_tests"]))
        reward = compute_reward(grade, format_ok=True, ran_execute=False)
        rows.append(
            {
                "task_id": p["task_id"],
                "prompt": prompt,
                "completion": completion,
                "pass_frac": reward["pass_frac"],
                "reward_total": reward["reward_total"],
                "passed": grade["passed"],
                "total": grade["total"],
            }
        )
    return rows