Spaces:
Sleeping
Sleeping
Download api/main.py from AbhiDS16/etl-debug-openenv: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/spaces/AbhiDS16/etl-debug-openenv/resolve/main/api/main.py
- Command line
-
hf download hf://spaces/AbhiDS16/etl-debug-openenv/api/main.py
-
curl -L -o main.py https://huggingface.co/spaces/AbhiDS16/etl-debug-openenv/resolve/main/api/main.py
10.4 kB
| """ | |
| api/main.py β FastAPI server for the Data Pipeline Incident Response OpenEnv. | |
| Endpoints (OpenEnv spec): | |
| POST /reset β Observation | |
| POST /step β StepResponse (observation, reward, done, info) | |
| GET /state β dict (internal state snapshot) | |
| GET /health β {"status": "ok", "env": "PipelineIncidentEnv"} | |
| GET /tasks β list of task IDs and descriptions | |
| GET /openenv.yaml β raw YAML spec file | |
| Session-based design: each session_id gets its own ETLDebugEnv instance, | |
| so concurrent judge evaluation runs don't corrupt each other's state. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import time | |
| import threading | |
| from pathlib import Path | |
| from typing import Any, Dict, Optional | |
| from fastapi import FastAPI, HTTPException | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.responses import PlainTextResponse | |
| from pydantic import BaseModel, Field | |
| from app.env import ETLDebugEnv | |
| from models.action import Action | |
| from models.observation import Observation | |
| from models.reward import Reward | |
| import subprocess, threading | |
| # ββ App setup βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| app = FastAPI( | |
| title="Data Pipeline Incident Response OpenEnv", | |
| description=( | |
| "An OpenEnv-compliant AI agent environment where the agent acts as an " | |
| "on-call data engineer triaging and fixing broken production pipelines. " | |
| "Implements the full OpenEnv spec: typed models, step/reset/state, openenv.yaml." | |
| ), | |
| version="1.1.0", | |
| docs_url="/docs", | |
| redoc_url="/redoc", | |
| ) | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| _START_TIME = time.time() | |
| # ββ Session registry (thread-safe) ββββββββββββββββββββββββββββββββββββββββββββ | |
| # Maps session_id β ETLDebugEnv instance. | |
| # This allows concurrent agent evaluations to run independently. | |
| _SESSIONS: Dict[str, ETLDebugEnv] = {} | |
| _SESSION_LOCK = threading.Lock() | |
| _DEFAULT_SESSION = "default" | |
| def _get_or_create_session(session_id: str) -> ETLDebugEnv: | |
| with _SESSION_LOCK: | |
| if session_id not in _SESSIONS: | |
| _SESSIONS[session_id] = ETLDebugEnv() | |
| return _SESSIONS[session_id] | |
| def _get_session(session_id: str) -> ETLDebugEnv: | |
| with _SESSION_LOCK: | |
| env = _SESSIONS.get(session_id) | |
| if env is None: | |
| raise HTTPException( | |
| status_code=400, | |
| detail=f"Session '{session_id}' not found. Call POST /reset first.", | |
| ) | |
| return env | |
| # ββ Request / response schemas ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class ResetRequest(BaseModel): | |
| task_id: str = Field( | |
| default="easy", | |
| description="Which task to load: 'easy', 'medium', 'hard', or 'cascade'.", | |
| ) | |
| session_id: str = Field( | |
| default=_DEFAULT_SESSION, | |
| description="Session identifier. Use unique IDs for concurrent evaluations.", | |
| ) | |
| class ResetResponse(BaseModel): | |
| observation: Observation | |
| task_id: str | |
| session_id: str | |
| message: str = "Environment reset successfully." | |
| class StepRequest(BaseModel): | |
| action: Action | |
| session_id: str = Field( | |
| default=_DEFAULT_SESSION, | |
| description="Must match the session_id used in /reset.", | |
| ) | |
| class StepResponse(BaseModel): | |
| observation: Observation | |
| reward: Reward | |
| done: bool | |
| info: Dict[str, Any] = Field(default_factory=dict) | |
| # ββ Endpoints βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def health() -> Dict[str, Any]: | |
| """Liveness check β automated ping target for HF Space validator.""" | |
| return { | |
| "status": "ok", | |
| "env": "PipelineIncidentEnv", | |
| "version": "1.1.0", | |
| "uptime_s": round(time.time() - _START_TIME, 1), | |
| "active_sessions": len(_SESSIONS), | |
| } | |
| def list_tasks() -> Dict[str, Any]: | |
| """List available tasks with descriptions and difficulty metadata.""" | |
| return { | |
| "tasks": [ | |
| { | |
| "task_id": "easy", | |
| "name": "INCIDENT #1044 β Type Chaos", | |
| "difficulty": "easy", | |
| "max_steps": 15, | |
| "description": ( | |
| "A SaaS users table was ingested by a broken CSV parser β " | |
| "every numeric column is the wrong type; salary has ~20% nulls. " | |
| "Fix type errors and fill nulls without dropping rows." | |
| ), | |
| "expected_score_range": [0.82, 0.95], | |
| }, | |
| { | |
| "task_id": "medium", | |
| "name": "INCIDENT #1891 β Schema Mismatch + Duplicates", | |
| "difficulty": "medium", | |
| "max_steps": 15, | |
| "description": ( | |
| "An orders table migrated from Node.js has camelCase columns, " | |
| "~20 duplicate rows, and order_amount stored as a string. " | |
| "Rename columns, deduplicate, and cast types." | |
| ), | |
| "expected_score_range": [0.68, 0.85], | |
| }, | |
| { | |
| "task_id": "hard", | |
| "name": "INCIDENT #2847 β Broken Join + Silent Data Loss", | |
| "difficulty": "hard", | |
| "max_steps": 20, | |
| "description": ( | |
| "BI dashboard is missing 15 orders due to an INNER JOIN. " | |
| "Key columns have different names (cust_id vs customer_id) " | |
| "AND different types (string vs int). Fix the join to preserve " | |
| "all 100 orders using LEFT JOIN." | |
| ), | |
| "expected_score_range": [0.45, 0.70], | |
| }, | |
| { | |
| "task_id": "cascade", | |
| "name": "INCIDENT #3091 β Cascading Pipeline Failure (P1)", | |
| "difficulty": "cascade", | |
| "max_steps": 25, | |
| "description": ( | |
| "Three tables (events β sessions β user_summary) have cascading " | |
| "failures: whitespace-padded event_type causes silent row drops, " | |
| "session_duration is a string, session_id has 10% nulls, and a " | |
| "type mismatch breaks the final user join. Must be fixed in " | |
| "dependency order to recover correct engagement metrics." | |
| ), | |
| "expected_score_range": [0.30, 0.60], | |
| }, | |
| ] | |
| } | |
| def run_inference(task: str = "all", max_steps: int = 15): | |
| """Trigger inference.py β called by the validator in Phase 2.""" | |
| def _run(): | |
| subprocess.run( | |
| ["python", "inference.py", "--task", task, "--max_steps", str(max_steps)], | |
| env=os.environ.copy(), | |
| cwd="/app", | |
| ) | |
| threading.Thread(target=_run, daemon=True).start() | |
| return {"status": "inference started", "task": task} | |
| def reset(request: ResetRequest = None) -> ResetResponse: | |
| if request is None: | |
| request = ResetRequest() | |
| """Reset the environment to the initial state for the given task.""" | |
| valid_tasks = ["easy", "medium", "hard", "cascade"] | |
| if request.task_id not in valid_tasks: | |
| raise HTTPException( | |
| status_code=422, | |
| detail=f"Unknown task_id '{request.task_id}'. Valid: {valid_tasks}", | |
| ) | |
| env = _get_or_create_session(request.session_id) | |
| try: | |
| obs = env.reset(task_id=request.task_id) | |
| except Exception as exc: | |
| raise HTTPException(status_code=500, detail=f"Reset failed: {exc}") from exc | |
| return ResetResponse( | |
| observation=obs, | |
| task_id=request.task_id, | |
| session_id=request.session_id, | |
| ) | |
| def step(request: StepRequest) -> StepResponse: | |
| """Apply one action to the environment.""" | |
| env = _get_session(request.session_id) | |
| if env._state is None: | |
| raise HTTPException( | |
| status_code=400, | |
| detail=f"Session '{request.session_id}' not initialised. Call POST /reset first.", | |
| ) | |
| try: | |
| obs, reward, done, info = env.step(request.action) | |
| except Exception as exc: | |
| raise HTTPException(status_code=500, detail=f"Step failed: {exc}") from exc | |
| return StepResponse( | |
| observation=obs, | |
| reward=reward, | |
| done=done, | |
| info=info, | |
| ) | |
| def get_state(session_id: str = _DEFAULT_SESSION) -> Dict[str, Any]: | |
| """Return a full snapshot of the internal episode state.""" | |
| try: | |
| env = _get_session(session_id) | |
| except HTTPException: | |
| return {"error": f"Session '{session_id}' not found. Call POST /reset first."} | |
| if env._state is None: | |
| return {"error": "No active episode. Call POST /reset first."} | |
| return env.state() | |
| def serve_openenv_yaml() -> str: | |
| """Serve the openenv.yaml spec file.""" | |
| yaml_path = Path(__file__).parent.parent / "openenv.yaml" | |
| if not yaml_path.exists(): | |
| raise HTTPException(status_code=404, detail="openenv.yaml not found.") | |
| return yaml_path.read_text() | |
| # ββ Startup event βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def startup_event() -> None: | |
| """Pre-warm the default session so the first /reset is instant.""" | |
| try: | |
| env = _get_or_create_session(_DEFAULT_SESSION) | |
| env.reset(task_id="easy") | |
| print("β Data Pipeline Incident Response OpenEnv started β pre-warmed on task 'easy'") | |
| except Exception as exc: | |
| print(f"β Pre-warm failed (non-fatal): {exc}") |