""" app/grader.py ============= Pure scoring function for the SQL Query Repair environment. Separated from env.py so it can be tested and called independently (e.g. from the /grader API endpoint after an episode completes). Score formula ------------- correctness = final_jaccard (0.0–1.0) turn_efficiency = 1 - (turns_used / max_turns) (0.0–1.0) clarification_quality = hits / total (1.0 if no clarifications asked) final_score = 0.60 * correctness + 0.25 * turn_efficiency + 0.15 * clarification_quality All outputs are deterministic and reproducible for identical inputs. """ from __future__ import annotations from app.models import GraderResult def compute_grade( final_jaccard: float, turns_used: int, max_turns: int, clarify_total: int, clarify_hits: int, solved: bool, ) -> GraderResult: """ Compute a GraderResult from raw episode statistics. Parameters ---------- final_jaccard : Jaccard score from the submit_query action (0.0 if the agent never submitted, or submitted an erroring query). turns_used : Number of turns consumed in the episode. max_turns : Maximum allowed turns for this difficulty. clarify_total : Total ask_clarification actions taken. clarify_hits : ask_clarification actions that received a real answer. solved : True if final_jaccard == 1.0. Returns ------- GraderResult with score in [0.0, 1.0] and sub-score breakdown. """ # -- Correctness -------------------------------------------------- correctness = round(float(final_jaccard), 4) # -- Turn efficiency ---------------------------------------------- if max_turns <= 0: turn_efficiency = 0.0 else: used_ratio = min(turns_used, max_turns) / max_turns turn_efficiency = round(1.0 - used_ratio, 4) # -- Clarification quality ---------------------------------------- if clarify_total == 0: # Agent asked nothing — neutral, doesn't penalise or reward clarification_quality = 1.0 else: clarification_quality = round(clarify_hits / clarify_total, 4) # -- Final weighted score ----------------------------------------- score = ( 0.60 * correctness + 0.25 * turn_efficiency + 0.15 * clarification_quality ) score = round(max(0.0, min(1.0, score)), 4) # -- Human-readable detail ---------------------------------------- detail = ( f"correctness={correctness:.2f} (×0.60) " f"+ efficiency={turn_efficiency:.2f} (×0.25) " f"+ clarify={clarification_quality:.2f} (×0.15) " f"= {score:.4f} | " f"turns={turns_used}/{max_turns} | " f"clarifications={clarify_hits}/{clarify_total} | " f"solved={'yes' if solved else 'no'}" ) return GraderResult( score=score, correctness=correctness, turn_efficiency=turn_efficiency, clarification_quality=clarification_quality, turns_used=turns_used, max_turns=max_turns, solved=solved, detail=detail, )