sql-repair-env / app /grader.py
ashoka0402
clean commit (removed token)
64ccc23
Raw History Blame Contribute Delete
3.2 kB
"""
app/grader.py
=============
Pure scoring function for the SQL Query Repair environment.
Separated from env.py so it can be tested and called independently
(e.g. from the /grader API endpoint after an episode completes).
Score formula
-------------
correctness = final_jaccard (0.0–1.0)
turn_efficiency = 1 - (turns_used / max_turns) (0.0–1.0)
clarification_quality = hits / total (1.0 if no clarifications asked)
final_score = 0.60 * correctness
+ 0.25 * turn_efficiency
+ 0.15 * clarification_quality
All outputs are deterministic and reproducible for identical inputs.
"""
from __future__ import annotations
from app.models import GraderResult
def compute_grade(
final_jaccard: float,
turns_used: int,
max_turns: int,
clarify_total: int,
clarify_hits: int,
solved: bool,
) -> GraderResult:
"""
Compute a GraderResult from raw episode statistics.
Parameters
----------
final_jaccard : Jaccard score from the submit_query action (0.0 if
the agent never submitted, or submitted an erroring query).
turns_used : Number of turns consumed in the episode.
max_turns : Maximum allowed turns for this difficulty.
clarify_total : Total ask_clarification actions taken.
clarify_hits : ask_clarification actions that received a real answer.
solved : True if final_jaccard == 1.0.
Returns
-------
GraderResult with score in [0.0, 1.0] and sub-score breakdown.
"""
# -- Correctness --------------------------------------------------
correctness = round(float(final_jaccard), 4)
# -- Turn efficiency ----------------------------------------------
if max_turns <= 0:
turn_efficiency = 0.0
else:
used_ratio = min(turns_used, max_turns) / max_turns
turn_efficiency = round(1.0 - used_ratio, 4)
# -- Clarification quality ----------------------------------------
if clarify_total == 0:
# Agent asked nothing — neutral, doesn't penalise or reward
clarification_quality = 1.0
else:
clarification_quality = round(clarify_hits / clarify_total, 4)
# -- Final weighted score -----------------------------------------
score = (
0.60 * correctness
+ 0.25 * turn_efficiency
+ 0.15 * clarification_quality
)
score = round(max(0.0, min(1.0, score)), 4)
# -- Human-readable detail ----------------------------------------
detail = (
f"correctness={correctness:.2f} (×0.60) "
f"+ efficiency={turn_efficiency:.2f} (×0.25) "
f"+ clarify={clarification_quality:.2f} (×0.15) "
f"= {score:.4f} | "
f"turns={turns_used}/{max_turns} | "
f"clarifications={clarify_hits}/{clarify_total} | "
f"solved={'yes' if solved else 'no'}"
)
return GraderResult(
score=score,
correctness=correctness,
turn_efficiency=turn_efficiency,
clarification_quality=clarification_quality,
turns_used=turns_used,
max_turns=max_turns,
solved=solved,
detail=detail,
)