Spaces:
Sleeping
Sleeping
Ajjack4 commited on
Commit ·
6c700b3
1
Parent(s): 87aa853
clamp all grader scores to strictly (0,1) exclusive per OpenEnv spec
Browse files- inference.py +2 -2
- tasks.py +17 -12
inference.py
CHANGED
|
@@ -273,8 +273,8 @@ async def run_task(client: OpenAI, env: EnvAdapter, task_id: str) -> None:
|
|
| 273 |
|
| 274 |
grader_data = env.grader(task_id)
|
| 275 |
grader_result = grader_data.get("grader_result", {})
|
| 276 |
-
score = float(grader_result.get("score", 0.
|
| 277 |
-
score =
|
| 278 |
success = score >= SUCCESS_SCORE_THRESHOLD
|
| 279 |
|
| 280 |
finally:
|
|
|
|
| 273 |
|
| 274 |
grader_data = env.grader(task_id)
|
| 275 |
grader_result = grader_data.get("grader_result", {})
|
| 276 |
+
score = float(grader_result.get("score", 0.001))
|
| 277 |
+
score = max(0.001, min(0.999, score)) # strictly (0, 1) exclusive
|
| 278 |
success = score >= SUCCESS_SCORE_THRESHOLD
|
| 279 |
|
| 280 |
finally:
|
tasks.py
CHANGED
|
@@ -41,14 +41,19 @@ TASKS = {
|
|
| 41 |
}
|
| 42 |
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
def grade_episode(task_id: str, env_state: dict) -> dict:
|
| 45 |
"""
|
| 46 |
-
Programmatic grader. Returns score 0
|
| 47 |
Called after episode ends (done=True).
|
| 48 |
"""
|
| 49 |
task = TASKS.get(task_id)
|
| 50 |
if not task:
|
| 51 |
-
return {"score": 0.0, "reason": f"Unknown task: {task_id}"}
|
| 52 |
|
| 53 |
services = env_state.get("services", {})
|
| 54 |
fixed_faults = set(env_state.get("fixed_faults", []))
|
|
@@ -69,21 +74,21 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
|
|
| 69 |
efficiency = max(0.0, 1.0 - (steps - 3) * 0.03)
|
| 70 |
score = min(1.0, 0.85 + efficiency * 0.15)
|
| 71 |
return {
|
| 72 |
-
"score":
|
| 73 |
"reason": f"All services healthy. Fixed in {steps} steps.",
|
| 74 |
"faults_fixed": faults_fixed,
|
| 75 |
"total_faults": total_faults,
|
| 76 |
}
|
| 77 |
elif faults_fixed > 0:
|
| 78 |
return {
|
| 79 |
-
"score": 0.4,
|
| 80 |
"reason": "Fault fixed but service not restarted or submit not called.",
|
| 81 |
"faults_fixed": faults_fixed,
|
| 82 |
"total_faults": total_faults,
|
| 83 |
}
|
| 84 |
else:
|
| 85 |
return {
|
| 86 |
-
"score": 0.0,
|
| 87 |
"reason": "No faults fixed.",
|
| 88 |
"faults_fixed": 0,
|
| 89 |
"total_faults": total_faults,
|
|
@@ -97,21 +102,21 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
|
|
| 97 |
efficiency = max(0.0, 1.0 - (steps - 5) * 0.02)
|
| 98 |
score = min(1.0, 0.85 + efficiency * 0.15)
|
| 99 |
return {
|
| 100 |
-
"score":
|
| 101 |
"reason": f"All services healthy. Both faults fixed in {steps} steps.",
|
| 102 |
"faults_fixed": faults_fixed,
|
| 103 |
"total_faults": total_faults,
|
| 104 |
}
|
| 105 |
elif faults_fixed == 1:
|
| 106 |
return {
|
| 107 |
-
"score": 0.45,
|
| 108 |
"reason": f"One of two faults fixed. Partial credit.",
|
| 109 |
"faults_fixed": faults_fixed,
|
| 110 |
"total_faults": total_faults,
|
| 111 |
}
|
| 112 |
else:
|
| 113 |
return {
|
| 114 |
-
"score": 0.0,
|
| 115 |
"reason": "No faults fixed.",
|
| 116 |
"faults_fixed": 0,
|
| 117 |
"total_faults": total_faults,
|
|
@@ -126,7 +131,7 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
|
|
| 126 |
efficiency = max(0.0, 1.0 - (steps - 4) * 0.04)
|
| 127 |
score = min(1.0, 0.80 + efficiency * 0.20)
|
| 128 |
return {
|
| 129 |
-
"score":
|
| 130 |
"reason": f"Root cause found and fixed. All services recovered in {steps} steps.",
|
| 131 |
"faults_fixed": faults_fixed,
|
| 132 |
"total_faults": total_faults,
|
|
@@ -134,17 +139,17 @@ def grade_episode(task_id: str, env_state: dict) -> dict:
|
|
| 134 |
elif faults_fixed > 0:
|
| 135 |
# Fixed something but not all healthy — partial
|
| 136 |
return {
|
| 137 |
-
"score": 0.3,
|
| 138 |
"reason": "Partial fix. Root cause addressed but services not fully recovered.",
|
| 139 |
"faults_fixed": faults_fixed,
|
| 140 |
"total_faults": total_faults,
|
| 141 |
}
|
| 142 |
else:
|
| 143 |
return {
|
| 144 |
-
"score": 0.0,
|
| 145 |
"reason": "Root cause not identified. All services still broken.",
|
| 146 |
"faults_fixed": 0,
|
| 147 |
"total_faults": total_faults,
|
| 148 |
}
|
| 149 |
|
| 150 |
-
return {"score": 0.0, "reason": "Unhandled task."}
|
|
|
|
| 41 |
}
|
| 42 |
|
| 43 |
|
| 44 |
+
def _clamp(score: float) -> float:
|
| 45 |
+
"""Clamp score to strictly (0, 1) exclusive as required by OpenEnv spec."""
|
| 46 |
+
return max(0.001, min(0.999, round(score, 3)))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
def grade_episode(task_id: str, env_state: dict) -> dict:
|
| 50 |
"""
|
| 51 |
+
Programmatic grader. Returns score strictly in (0, 1) exclusive.
|
| 52 |
Called after episode ends (done=True).
|
| 53 |
"""
|
| 54 |
task = TASKS.get(task_id)
|
| 55 |
if not task:
|
| 56 |
+
return {"score": _clamp(0.0), "reason": f"Unknown task: {task_id}"}
|
| 57 |
|
| 58 |
services = env_state.get("services", {})
|
| 59 |
fixed_faults = set(env_state.get("fixed_faults", []))
|
|
|
|
| 74 |
efficiency = max(0.0, 1.0 - (steps - 3) * 0.03)
|
| 75 |
score = min(1.0, 0.85 + efficiency * 0.15)
|
| 76 |
return {
|
| 77 |
+
"score": _clamp(score),
|
| 78 |
"reason": f"All services healthy. Fixed in {steps} steps.",
|
| 79 |
"faults_fixed": faults_fixed,
|
| 80 |
"total_faults": total_faults,
|
| 81 |
}
|
| 82 |
elif faults_fixed > 0:
|
| 83 |
return {
|
| 84 |
+
"score": _clamp(0.4),
|
| 85 |
"reason": "Fault fixed but service not restarted or submit not called.",
|
| 86 |
"faults_fixed": faults_fixed,
|
| 87 |
"total_faults": total_faults,
|
| 88 |
}
|
| 89 |
else:
|
| 90 |
return {
|
| 91 |
+
"score": _clamp(0.0),
|
| 92 |
"reason": "No faults fixed.",
|
| 93 |
"faults_fixed": 0,
|
| 94 |
"total_faults": total_faults,
|
|
|
|
| 102 |
efficiency = max(0.0, 1.0 - (steps - 5) * 0.02)
|
| 103 |
score = min(1.0, 0.85 + efficiency * 0.15)
|
| 104 |
return {
|
| 105 |
+
"score": _clamp(score),
|
| 106 |
"reason": f"All services healthy. Both faults fixed in {steps} steps.",
|
| 107 |
"faults_fixed": faults_fixed,
|
| 108 |
"total_faults": total_faults,
|
| 109 |
}
|
| 110 |
elif faults_fixed == 1:
|
| 111 |
return {
|
| 112 |
+
"score": _clamp(0.45),
|
| 113 |
"reason": f"One of two faults fixed. Partial credit.",
|
| 114 |
"faults_fixed": faults_fixed,
|
| 115 |
"total_faults": total_faults,
|
| 116 |
}
|
| 117 |
else:
|
| 118 |
return {
|
| 119 |
+
"score": _clamp(0.0),
|
| 120 |
"reason": "No faults fixed.",
|
| 121 |
"faults_fixed": 0,
|
| 122 |
"total_faults": total_faults,
|
|
|
|
| 131 |
efficiency = max(0.0, 1.0 - (steps - 4) * 0.04)
|
| 132 |
score = min(1.0, 0.80 + efficiency * 0.20)
|
| 133 |
return {
|
| 134 |
+
"score": _clamp(score),
|
| 135 |
"reason": f"Root cause found and fixed. All services recovered in {steps} steps.",
|
| 136 |
"faults_fixed": faults_fixed,
|
| 137 |
"total_faults": total_faults,
|
|
|
|
| 139 |
elif faults_fixed > 0:
|
| 140 |
# Fixed something but not all healthy — partial
|
| 141 |
return {
|
| 142 |
+
"score": _clamp(0.3),
|
| 143 |
"reason": "Partial fix. Root cause addressed but services not fully recovered.",
|
| 144 |
"faults_fixed": faults_fixed,
|
| 145 |
"total_faults": total_faults,
|
| 146 |
}
|
| 147 |
else:
|
| 148 |
return {
|
| 149 |
+
"score": _clamp(0.0),
|
| 150 |
"reason": "Root cause not identified. All services still broken.",
|
| 151 |
"faults_fixed": 0,
|
| 152 |
"total_faults": total_faults,
|
| 153 |
}
|
| 154 |
|
| 155 |
+
return {"score": _clamp(0.0), "reason": "Unhandled task."}
|