Spaces:
Sleeping
Sleeping
| """ | |
| Task definitions and programmatic graders. | |
| Each task defines faults to inject and a grader that scores strictly in (0, 1). | |
| """ | |
| TASKS = { | |
| "task1": { | |
| "id": "task1", | |
| "name": "Single Service Fault", | |
| "difficulty": "easy", | |
| "description": ( | |
| "One service is down due to a single misconfigured environment variable. " | |
| "The API cannot connect to the database because DB_PORT is set to the wrong value. " | |
| "Read the logs, find the bad config, fix it, and restart the service." | |
| ), | |
| "faults": ["wrong_db_port"], | |
| }, | |
| "task2": { | |
| "id": "task2", | |
| "name": "Multi-Fault Diagnosis", | |
| "difficulty": "medium", | |
| "description": ( | |
| "Two services are failing due to independent faults. " | |
| "The API has a wrong database port AND the worker is missing its queue URL. " | |
| "Both must be fixed for full score. Partial credit for fixing one." | |
| ), | |
| "faults": ["wrong_db_port", "missing_queue_url"], | |
| }, | |
| "task3": { | |
| "id": "task3", | |
| "name": "Cascading Failure Debug", | |
| "difficulty": "hard", | |
| "description": ( | |
| "The database has wrong credentials, causing the API to log a misleading " | |
| "'connection timeout' error, which cascades to the worker and proxy. " | |
| "Logs are misleading — the agent must trace the root cause (DB_PASSWORD on the " | |
| "database service) rather than patching downstream symptoms." | |
| ), | |
| "faults": ["wrong_db_password"], | |
| }, | |
| "task4": { | |
| "id": "task4", | |
| "name": "Triple Fault Storm", | |
| "difficulty": "hard", | |
| "description": ( | |
| "Three independent faults hit at once: wrong DB_HOST on the API, " | |
| "wrong API_HOST on the worker, and wrong PROXY_UPSTREAM on the proxy. " | |
| "Each service is broken for a different reason. Agent must diagnose and fix all three " | |
| "without being misled by cascading errors." | |
| ), | |
| "faults": ["wrong_db_host", "wrong_api_host", "wrong_api_port"], | |
| }, | |
| "task5": { | |
| "id": "task5", | |
| "name": "Silent Misconfiguration", | |
| "difficulty": "expert", | |
| "description": ( | |
| "Two subtle faults with no obvious error messages: DB_PASSWORD is missing on the API " | |
| "(not wrong — missing entirely), and the worker's API_HOST points to the wrong port. " | |
| "Services appear to start but immediately fail silently. " | |
| "The agent must inspect env vars carefully rather than relying on logs alone." | |
| ), | |
| "faults": ["missing_db_password", "wrong_api_host"], | |
| }, | |
| } | |
| def _clamp(score: float) -> float: | |
| """Clamp score to strictly (0, 1) exclusive as required by OpenEnv spec.""" | |
| return max(0.001, min(0.999, round(score, 3))) | |
| def grade_episode(task_id: str, env_state: dict) -> dict: | |
| """ | |
| Programmatic grader. Returns score strictly in (0, 1) exclusive. | |
| Called after episode ends (done=True). | |
| """ | |
| task = TASKS.get(task_id) | |
| if not task: | |
| return {"score": _clamp(0.001), "reason": f"Unknown task: {task_id}"} | |
| services = env_state.get("services", {}) | |
| fixed_faults = set(env_state.get("fixed_faults", [])) | |
| active_faults = set(env_state.get("active_faults", [])) | |
| steps = env_state.get("step", 0) | |
| all_healthy = all(s["status"] == "healthy" for s in services.values()) | |
| total_faults = len(active_faults) | |
| faults_fixed = len(fixed_faults) | |
| # --------------------------------------------------------------- | |
| # Task 1 — single fault, efficiency bonus | |
| # --------------------------------------------------------------- | |
| if task_id == "task1": | |
| if all_healthy and faults_fixed == total_faults: | |
| efficiency = max(0.0, 1.0 - max(0, steps - 4) * 0.04) | |
| score = 0.75 + efficiency * 0.20 | |
| return { | |
| "score": _clamp(score), | |
| "reason": f"All services healthy. Fixed in {steps} steps.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed > 0: | |
| return { | |
| "score": _clamp(0.35), | |
| "reason": "Fault fixed but episode not completed correctly.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| return { | |
| "score": _clamp(0.05), | |
| "reason": "No faults fixed.", | |
| "faults_fixed": 0, | |
| "total_faults": total_faults, | |
| } | |
| # --------------------------------------------------------------- | |
| # Task 2 — partial credit per fault fixed | |
| # --------------------------------------------------------------- | |
| if task_id == "task2": | |
| if all_healthy and faults_fixed == total_faults: | |
| efficiency = max(0.0, 1.0 - max(0, steps - 6) * 0.03) | |
| score = 0.70 + efficiency * 0.25 | |
| return { | |
| "score": _clamp(score), | |
| "reason": f"Both faults fixed. All services healthy in {steps} steps.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed == 1: | |
| return { | |
| "score": _clamp(0.40), | |
| "reason": "One of two faults fixed. Partial credit.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| return { | |
| "score": _clamp(0.05), | |
| "reason": "No faults fixed.", | |
| "faults_fixed": 0, | |
| "total_faults": total_faults, | |
| } | |
| # --------------------------------------------------------------- | |
| # Task 3 — cascading root cause | |
| # --------------------------------------------------------------- | |
| if task_id == "task3": | |
| if all_healthy and faults_fixed == total_faults: | |
| efficiency = max(0.0, 1.0 - max(0, steps - 5) * 0.04) | |
| score = 0.70 + efficiency * 0.25 | |
| return { | |
| "score": _clamp(score), | |
| "reason": f"Root cause found and fixed. All services recovered in {steps} steps.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed > 0: | |
| return { | |
| "score": _clamp(0.30), | |
| "reason": "Root cause addressed but services not fully recovered.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| return { | |
| "score": _clamp(0.05), | |
| "reason": "Root cause not identified.", | |
| "faults_fixed": 0, | |
| "total_faults": total_faults, | |
| } | |
| # --------------------------------------------------------------- | |
| # Task 4 — triple fault storm (partial credit per fault) | |
| # --------------------------------------------------------------- | |
| if task_id == "task4": | |
| if all_healthy and faults_fixed == total_faults: | |
| efficiency = max(0.0, 1.0 - max(0, steps - 10) * 0.025) | |
| score = 0.65 + efficiency * 0.30 | |
| return { | |
| "score": _clamp(score), | |
| "reason": f"All 3 faults fixed. All services healthy in {steps} steps.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed == 2: | |
| return { | |
| "score": _clamp(0.55), | |
| "reason": "2 of 3 faults fixed.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed == 1: | |
| return { | |
| "score": _clamp(0.28), | |
| "reason": "1 of 3 faults fixed.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| return { | |
| "score": _clamp(0.05), | |
| "reason": "No faults fixed.", | |
| "faults_fixed": 0, | |
| "total_faults": total_faults, | |
| } | |
| # --------------------------------------------------------------- | |
| # Task 5 — silent misconfiguration (expert) | |
| # --------------------------------------------------------------- | |
| if task_id == "task5": | |
| if all_healthy and faults_fixed == total_faults: | |
| efficiency = max(0.0, 1.0 - max(0, steps - 8) * 0.03) | |
| score = 0.65 + efficiency * 0.30 | |
| return { | |
| "score": _clamp(score), | |
| "reason": f"Both silent faults found and fixed in {steps} steps. Expert diagnosis.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| elif faults_fixed == 1: | |
| return { | |
| "score": _clamp(0.38), | |
| "reason": "One silent fault found. Second still undetected.", | |
| "faults_fixed": faults_fixed, | |
| "total_faults": total_faults, | |
| } | |
| return { | |
| "score": _clamp(0.05), | |
| "reason": "No faults found. Agent must inspect env vars, not just logs.", | |
| "faults_fixed": 0, | |
| "total_faults": total_faults, | |
| } | |
| return {"score": _clamp(0.05), "reason": "Unhandled task."} | |