Spaces:
Sleeping
Sleeping
Download app/grader.py from skillissue04/sql-repair-env: direct link, hf CLI and curl.
- Browser
- Download file 3.2 kB
-
https://huggingface.co/spaces/skillissue04/sql-repair-env/resolve/main/app/grader.py
- Command line
-
hf download hf://spaces/skillissue04/sql-repair-env/app/grader.py
-
curl -L -o grader.py https://huggingface.co/spaces/skillissue04/sql-repair-env/resolve/main/app/grader.py
3.2 kB
| """ | |
| app/grader.py | |
| ============= | |
| Pure scoring function for the SQL Query Repair environment. | |
| Separated from env.py so it can be tested and called independently | |
| (e.g. from the /grader API endpoint after an episode completes). | |
| Score formula | |
| ------------- | |
| correctness = final_jaccard (0.0–1.0) | |
| turn_efficiency = 1 - (turns_used / max_turns) (0.0–1.0) | |
| clarification_quality = hits / total (1.0 if no clarifications asked) | |
| final_score = 0.60 * correctness | |
| + 0.25 * turn_efficiency | |
| + 0.15 * clarification_quality | |
| All outputs are deterministic and reproducible for identical inputs. | |
| """ | |
| from __future__ import annotations | |
| from app.models import GraderResult | |
| def compute_grade( | |
| final_jaccard: float, | |
| turns_used: int, | |
| max_turns: int, | |
| clarify_total: int, | |
| clarify_hits: int, | |
| solved: bool, | |
| ) -> GraderResult: | |
| """ | |
| Compute a GraderResult from raw episode statistics. | |
| Parameters | |
| ---------- | |
| final_jaccard : Jaccard score from the submit_query action (0.0 if | |
| the agent never submitted, or submitted an erroring query). | |
| turns_used : Number of turns consumed in the episode. | |
| max_turns : Maximum allowed turns for this difficulty. | |
| clarify_total : Total ask_clarification actions taken. | |
| clarify_hits : ask_clarification actions that received a real answer. | |
| solved : True if final_jaccard == 1.0. | |
| Returns | |
| ------- | |
| GraderResult with score in [0.0, 1.0] and sub-score breakdown. | |
| """ | |
| # -- Correctness -------------------------------------------------- | |
| correctness = round(float(final_jaccard), 4) | |
| # -- Turn efficiency ---------------------------------------------- | |
| if max_turns <= 0: | |
| turn_efficiency = 0.0 | |
| else: | |
| used_ratio = min(turns_used, max_turns) / max_turns | |
| turn_efficiency = round(1.0 - used_ratio, 4) | |
| # -- Clarification quality ---------------------------------------- | |
| if clarify_total == 0: | |
| # Agent asked nothing — neutral, doesn't penalise or reward | |
| clarification_quality = 1.0 | |
| else: | |
| clarification_quality = round(clarify_hits / clarify_total, 4) | |
| # -- Final weighted score ----------------------------------------- | |
| score = ( | |
| 0.60 * correctness | |
| + 0.25 * turn_efficiency | |
| + 0.15 * clarification_quality | |
| ) | |
| score = round(max(0.0, min(1.0, score)), 4) | |
| # -- Human-readable detail ---------------------------------------- | |
| detail = ( | |
| f"correctness={correctness:.2f} (×0.60) " | |
| f"+ efficiency={turn_efficiency:.2f} (×0.25) " | |
| f"+ clarify={clarification_quality:.2f} (×0.15) " | |
| f"= {score:.4f} | " | |
| f"turns={turns_used}/{max_turns} | " | |
| f"clarifications={clarify_hits}/{clarify_total} | " | |
| f"solved={'yes' if solved else 'no'}" | |
| ) | |
| return GraderResult( | |
| score=score, | |
| correctness=correctness, | |
| turn_efficiency=turn_efficiency, | |
| clarification_quality=clarification_quality, | |
| turns_used=turns_used, | |
| max_turns=max_turns, | |
| solved=solved, | |
| detail=detail, | |
| ) | |