Ajjack4 commited on
Commit
6c700b3
·
1 Parent(s): 87aa853

clamp all grader scores to strictly (0,1) exclusive per OpenEnv spec

Browse files
Files changed (2) hide show
  1. inference.py +2 -2
  2. tasks.py +17 -12
inference.py CHANGED
@@ -273,8 +273,8 @@ async def run_task(client: OpenAI, env: EnvAdapter, task_id: str) -> None:
273
 
274
  grader_data = env.grader(task_id)
275
  grader_result = grader_data.get("grader_result", {})
276
- score = float(grader_result.get("score", 0.0))
277
- score = min(max(score, 0.0), 1.0)
278
  success = score >= SUCCESS_SCORE_THRESHOLD
279
 
280
  finally:
 
273
 
274
  grader_data = env.grader(task_id)
275
  grader_result = grader_data.get("grader_result", {})
276
+ score = float(grader_result.get("score", 0.001))
277
+ score = max(0.001, min(0.999, score)) # strictly (0, 1) exclusive
278
  success = score >= SUCCESS_SCORE_THRESHOLD
279
 
280
  finally:
tasks.py CHANGED
@@ -41,14 +41,19 @@ TASKS = {
41
  }
42
 
43
 
 
 
 
 
 
44
  def grade_episode(task_id: str, env_state: dict) -> dict:
45
  """
46
- Programmatic grader. Returns score 0.0-1.0 and breakdown.
47
  Called after episode ends (done=True).
48
  """
49
  task = TASKS.get(task_id)
50
  if not task:
51
- return {"score": 0.0, "reason": f"Unknown task: {task_id}"}
52
 
53
  services = env_state.get("services", {})
54
  fixed_faults = set(env_state.get("fixed_faults", []))
@@ -69,21 +74,21 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
69
  efficiency = max(0.0, 1.0 - (steps - 3) * 0.03)
70
  score = min(1.0, 0.85 + efficiency * 0.15)
71
  return {
72
- "score": round(score, 3),
73
  "reason": f"All services healthy. Fixed in {steps} steps.",
74
  "faults_fixed": faults_fixed,
75
  "total_faults": total_faults,
76
  }
77
  elif faults_fixed > 0:
78
  return {
79
- "score": 0.4,
80
  "reason": "Fault fixed but service not restarted or submit not called.",
81
  "faults_fixed": faults_fixed,
82
  "total_faults": total_faults,
83
  }
84
  else:
85
  return {
86
- "score": 0.0,
87
  "reason": "No faults fixed.",
88
  "faults_fixed": 0,
89
  "total_faults": total_faults,
@@ -97,21 +102,21 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
97
  efficiency = max(0.0, 1.0 - (steps - 5) * 0.02)
98
  score = min(1.0, 0.85 + efficiency * 0.15)
99
  return {
100
- "score": round(score, 3),
101
  "reason": f"All services healthy. Both faults fixed in {steps} steps.",
102
  "faults_fixed": faults_fixed,
103
  "total_faults": total_faults,
104
  }
105
  elif faults_fixed == 1:
106
  return {
107
- "score": 0.45,
108
  "reason": f"One of two faults fixed. Partial credit.",
109
  "faults_fixed": faults_fixed,
110
  "total_faults": total_faults,
111
  }
112
  else:
113
  return {
114
- "score": 0.0,
115
  "reason": "No faults fixed.",
116
  "faults_fixed": 0,
117
  "total_faults": total_faults,
@@ -126,7 +131,7 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
126
  efficiency = max(0.0, 1.0 - (steps - 4) * 0.04)
127
  score = min(1.0, 0.80 + efficiency * 0.20)
128
  return {
129
- "score": round(score, 3),
130
  "reason": f"Root cause found and fixed. All services recovered in {steps} steps.",
131
  "faults_fixed": faults_fixed,
132
  "total_faults": total_faults,
@@ -134,17 +139,17 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
134
  elif faults_fixed > 0:
135
  # Fixed something but not all healthy — partial
136
  return {
137
- "score": 0.3,
138
  "reason": "Partial fix. Root cause addressed but services not fully recovered.",
139
  "faults_fixed": faults_fixed,
140
  "total_faults": total_faults,
141
  }
142
  else:
143
  return {
144
- "score": 0.0,
145
  "reason": "Root cause not identified. All services still broken.",
146
  "faults_fixed": 0,
147
  "total_faults": total_faults,
148
  }
149
 
150
- return {"score": 0.0, "reason": "Unhandled task."}
 
41
  }
42
 
43
 
44
+ def _clamp(score: float) -> float:
45
+ """Clamp score to strictly (0, 1) exclusive as required by OpenEnv spec."""
46
+ return max(0.001, min(0.999, round(score, 3)))
47
+
48
+
49
  def grade_episode(task_id: str, env_state: dict) -> dict:
50
  """
51
+ Programmatic grader. Returns score strictly in (0, 1) exclusive.
52
  Called after episode ends (done=True).
53
  """
54
  task = TASKS.get(task_id)
55
  if not task:
56
+ return {"score": _clamp(0.0), "reason": f"Unknown task: {task_id}"}
57
 
58
  services = env_state.get("services", {})
59
  fixed_faults = set(env_state.get("fixed_faults", []))
 
74
  efficiency = max(0.0, 1.0 - (steps - 3) * 0.03)
75
  score = min(1.0, 0.85 + efficiency * 0.15)
76
  return {
77
+ "score": _clamp(score),
78
  "reason": f"All services healthy. Fixed in {steps} steps.",
79
  "faults_fixed": faults_fixed,
80
  "total_faults": total_faults,
81
  }
82
  elif faults_fixed > 0:
83
  return {
84
+ "score": _clamp(0.4),
85
  "reason": "Fault fixed but service not restarted or submit not called.",
86
  "faults_fixed": faults_fixed,
87
  "total_faults": total_faults,
88
  }
89
  else:
90
  return {
91
+ "score": _clamp(0.0),
92
  "reason": "No faults fixed.",
93
  "faults_fixed": 0,
94
  "total_faults": total_faults,
 
102
  efficiency = max(0.0, 1.0 - (steps - 5) * 0.02)
103
  score = min(1.0, 0.85 + efficiency * 0.15)
104
  return {
105
+ "score": _clamp(score),
106
  "reason": f"All services healthy. Both faults fixed in {steps} steps.",
107
  "faults_fixed": faults_fixed,
108
  "total_faults": total_faults,
109
  }
110
  elif faults_fixed == 1:
111
  return {
112
+ "score": _clamp(0.45),
113
  "reason": f"One of two faults fixed. Partial credit.",
114
  "faults_fixed": faults_fixed,
115
  "total_faults": total_faults,
116
  }
117
  else:
118
  return {
119
+ "score": _clamp(0.0),
120
  "reason": "No faults fixed.",
121
  "faults_fixed": 0,
122
  "total_faults": total_faults,
 
131
  efficiency = max(0.0, 1.0 - (steps - 4) * 0.04)
132
  score = min(1.0, 0.80 + efficiency * 0.20)
133
  return {
134
+ "score": _clamp(score),
135
  "reason": f"Root cause found and fixed. All services recovered in {steps} steps.",
136
  "faults_fixed": faults_fixed,
137
  "total_faults": total_faults,
 
139
  elif faults_fixed > 0:
140
  # Fixed something but not all healthy — partial
141
  return {
142
+ "score": _clamp(0.3),
143
  "reason": "Partial fix. Root cause addressed but services not fully recovered.",
144
  "faults_fixed": faults_fixed,
145
  "total_faults": total_faults,
146
  }
147
  else:
148
  return {
149
+ "score": _clamp(0.0),
150
  "reason": "Root cause not identified. All services still broken.",
151
  "faults_fixed": 0,
152
  "total_faults": total_faults,
153
  }
154
 
155
+ return {"score": _clamp(0.0), "reason": "Unhandled task."}