Spaces:
Sleeping
Sleeping
Commit ·
7eba106
1
Parent(s): 1818234
Critical: fix grader wrapper to return tuple (reward, error_msg, bugs_fixed) with clamped reward
Browse files- server/graders/grader_api.py +18 -18
- server/tasks/task_registry.py +3 -2
server/graders/grader_api.py
CHANGED
|
@@ -2,11 +2,10 @@
|
|
| 2 |
|
| 3 |
OpenEnv validator expects graders to:
|
| 4 |
1. Be callable
|
| 5 |
-
2. Return float
|
| 6 |
-
3. Return
|
| 7 |
|
| 8 |
-
Our graders return
|
| 9 |
-
This wrapper extracts just the float reward.
|
| 10 |
"""
|
| 11 |
|
| 12 |
from server.graders.json_grader import grade_task1 as _g1
|
|
@@ -14,27 +13,28 @@ from server.graders.yaml_grader import grade_task2 as _g2
|
|
| 14 |
from server.graders.dockerfile_grader import grade_task3 as _g3
|
| 15 |
|
| 16 |
|
| 17 |
-
def
|
| 18 |
-
"""Extract
|
| 19 |
-
|
| 20 |
if isinstance(result, tuple):
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
def grade_task1(x):
|
| 29 |
-
"""Task 1 (JSON) grader wrapper."""
|
| 30 |
-
return
|
| 31 |
|
| 32 |
|
| 33 |
def grade_task2(x):
|
| 34 |
-
"""Task 2 (YAML) grader wrapper."""
|
| 35 |
-
return
|
| 36 |
|
| 37 |
|
| 38 |
def grade_task3(x):
|
| 39 |
-
"""Task 3 (Dockerfile) grader wrapper."""
|
| 40 |
-
return
|
|
|
|
| 2 |
|
| 3 |
OpenEnv validator expects graders to:
|
| 4 |
1. Be callable
|
| 5 |
+
2. Return tuple: (reward: float, error_msg: str, bugs_fixed: list)
|
| 6 |
+
3. Return reward in [0.0, 1.0] range (clamped to avoid exact 0/1)
|
| 7 |
|
| 8 |
+
Our graders already return this format, but we clamp rewards here.
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from server.graders.json_grader import grade_task1 as _g1
|
|
|
|
| 13 |
from server.graders.dockerfile_grader import grade_task3 as _g3
|
| 14 |
|
| 15 |
|
| 16 |
+
def _clamp_reward(result):
|
| 17 |
+
"""Extract and clamp reward from grader result tuple.
|
| 18 |
+
Returns the original tuple with clamped reward."""
|
| 19 |
if isinstance(result, tuple):
|
| 20 |
+
reward, error_msg, bugs_fixed = result
|
| 21 |
+
# Clamp to strict (0.001, 0.999) range to avoid exact boundaries
|
| 22 |
+
clamped_reward = max(0.001, min(0.999, float(reward)))
|
| 23 |
+
return clamped_reward, error_msg, bugs_fixed
|
| 24 |
+
# Fallback (shouldn't happen)
|
| 25 |
+
return result
|
| 26 |
|
| 27 |
|
| 28 |
def grade_task1(x):
|
| 29 |
+
"""Task 1 (JSON) grader wrapper - returns tuple with clamped reward."""
|
| 30 |
+
return _clamp_reward(_g1(x))
|
| 31 |
|
| 32 |
|
| 33 |
def grade_task2(x):
|
| 34 |
+
"""Task 2 (YAML) grader wrapper - returns tuple with clamped reward."""
|
| 35 |
+
return _clamp_reward(_g2(x))
|
| 36 |
|
| 37 |
|
| 38 |
def grade_task3(x):
|
| 39 |
+
"""Task 3 (Dockerfile) grader wrapper - returns tuple with clamped reward."""
|
| 40 |
+
return _clamp_reward(_g3(x))
|
server/tasks/task_registry.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
from typing import Callable, Dict, List
|
| 2 |
|
| 3 |
from server.tasks import task1_json, task2_yaml, task3_dockerfile
|
| 4 |
from server.graders.grader_api import grade_task1, grade_task2, grade_task3
|
|
@@ -14,7 +14,8 @@ class TaskInfo:
|
|
| 14 |
self.broken_config: str = module.BROKEN_CONFIG
|
| 15 |
self.error_message: str = module.ERROR_MESSAGE
|
| 16 |
self.ground_truth: str = module.GROUND_TRUTH
|
| 17 |
-
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
TASK_ORDER = [
|
|
|
|
| 1 |
+
from typing import Callable, Dict, List, Tuple
|
| 2 |
|
| 3 |
from server.tasks import task1_json, task2_yaml, task3_dockerfile
|
| 4 |
from server.graders.grader_api import grade_task1, grade_task2, grade_task3
|
|
|
|
| 14 |
self.broken_config: str = module.BROKEN_CONFIG
|
| 15 |
self.error_message: str = module.ERROR_MESSAGE
|
| 16 |
self.ground_truth: str = module.GROUND_TRUTH
|
| 17 |
+
# Grader returns (reward: float, error_msg: str, bugs_fixed: list)
|
| 18 |
+
self.grader: Callable[[str], Tuple[float, str, List[str]]] = grader_func
|
| 19 |
|
| 20 |
|
| 21 |
TASK_ORDER = [
|