NanduKondreddy commited on
Commit
7eba106
·
1 Parent(s): 1818234

Critical: fix grader wrapper to return tuple (reward, error_msg, bugs_fixed) with clamped reward

Browse files
server/graders/grader_api.py CHANGED
@@ -2,11 +2,10 @@
2
 
3
  OpenEnv validator expects graders to:
4
  1. Be callable
5
- 2. Return float only (not tuple)
6
- 3. Return values in [0.0, 1.0] range
7
 
8
- Our graders return (reward, error_msg, bugs_fixed).
9
- This wrapper extracts just the float reward.
10
  """
11
 
12
  from server.graders.json_grader import grade_task1 as _g1
@@ -14,27 +13,28 @@ from server.graders.yaml_grader import grade_task2 as _g2
14
  from server.graders.dockerfile_grader import grade_task3 as _g3
15
 
16
 
17
- def _extract(result):
18
- """Extract float reward from grader result tuple or return as-is if already float.
19
- Clamp to strict (0.001, 0.999) range for validator compatibility."""
20
  if isinstance(result, tuple):
21
- r = float(result[0])
22
- else:
23
- r = float(result)
24
- # Strict bounds: not 0.0, not 1.0
25
- return max(0.001, min(0.999, r))
 
26
 
27
 
28
  def grade_task1(x):
29
- """Task 1 (JSON) grader wrapper."""
30
- return _extract(_g1(x))
31
 
32
 
33
  def grade_task2(x):
34
- """Task 2 (YAML) grader wrapper."""
35
- return _extract(_g2(x))
36
 
37
 
38
  def grade_task3(x):
39
- """Task 3 (Dockerfile) grader wrapper."""
40
- return _extract(_g3(x))
 
2
 
3
  OpenEnv validator expects graders to:
4
  1. Be callable
5
+ 2. Return tuple: (reward: float, error_msg: str, bugs_fixed: list)
6
+ 3. Return reward in [0.0, 1.0] range (clamped to avoid exact 0/1)
7
 
8
+ Our graders already return this format, but we clamp rewards here.
 
9
  """
10
 
11
  from server.graders.json_grader import grade_task1 as _g1
 
13
  from server.graders.dockerfile_grader import grade_task3 as _g3
14
 
15
 
16
+ def _clamp_reward(result):
17
+ """Extract and clamp reward from grader result tuple.
18
+ Returns the original tuple with clamped reward."""
19
  if isinstance(result, tuple):
20
+ reward, error_msg, bugs_fixed = result
21
+ # Clamp to strict (0.001, 0.999) range to avoid exact boundaries
22
+ clamped_reward = max(0.001, min(0.999, float(reward)))
23
+ return clamped_reward, error_msg, bugs_fixed
24
+ # Fallback (shouldn't happen)
25
+ return result
26
 
27
 
28
  def grade_task1(x):
29
+ """Task 1 (JSON) grader wrapper - returns tuple with clamped reward."""
30
+ return _clamp_reward(_g1(x))
31
 
32
 
33
  def grade_task2(x):
34
+ """Task 2 (YAML) grader wrapper - returns tuple with clamped reward."""
35
+ return _clamp_reward(_g2(x))
36
 
37
 
38
  def grade_task3(x):
39
+ """Task 3 (Dockerfile) grader wrapper - returns tuple with clamped reward."""
40
+ return _clamp_reward(_g3(x))
server/tasks/task_registry.py CHANGED
@@ -1,4 +1,4 @@
1
- from typing import Callable, Dict, List
2
 
3
  from server.tasks import task1_json, task2_yaml, task3_dockerfile
4
  from server.graders.grader_api import grade_task1, grade_task2, grade_task3
@@ -14,7 +14,8 @@ class TaskInfo:
14
  self.broken_config: str = module.BROKEN_CONFIG
15
  self.error_message: str = module.ERROR_MESSAGE
16
  self.ground_truth: str = module.GROUND_TRUTH
17
- self.grader: Callable[[str], float] = grader_func
 
18
 
19
 
20
  TASK_ORDER = [
 
1
+ from typing import Callable, Dict, List, Tuple
2
 
3
  from server.tasks import task1_json, task2_yaml, task3_dockerfile
4
  from server.graders.grader_api import grade_task1, grade_task2, grade_task3
 
14
  self.broken_config: str = module.BROKEN_CONFIG
15
  self.error_message: str = module.ERROR_MESSAGE
16
  self.ground_truth: str = module.GROUND_TRUTH
17
+ # Grader returns (reward: float, error_msg: str, bugs_fixed: list)
18
+ self.grader: Callable[[str], Tuple[float, str, List[str]]] = grader_func
19
 
20
 
21
  TASK_ORDER = [