Spaces:
Sleeping
Sleeping
Download scripts/phase_readiness.py from Kyoiske/incident_ops_env: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/spaces/Kyoiske/incident_ops_env/resolve/main/scripts/phase_readiness.py
- Command line
-
hf download hf://spaces/Kyoiske/incident_ops_env/scripts/phase_readiness.py
-
curl -L -o phase_readiness.py https://huggingface.co/spaces/Kyoiske/incident_ops_env/resolve/main/scripts/phase_readiness.py
10.1 kB
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| from dataclasses import dataclass | |
| import httpx | |
| from fastapi.testclient import TestClient | |
| from baseline import run_baseline_sync | |
| from incident_ops_env.server.app import app | |
| from incident_ops_env.server.scenario_loader import list_scenarios_for_task | |
| class CheckResult: | |
| name: str | |
| passed: bool | |
| detail: str | |
| def check_openenv_contract() -> CheckResult: | |
| client = TestClient(app) | |
| required = ["/health", "/tasks", "/reset", "/step", "/state", "/grader", "/baseline"] | |
| found = {route.path for route in app.routes} | |
| missing = [path for path in required if path not in found] | |
| if missing: | |
| return CheckResult("OpenEnv endpoint contract", False, f"Missing endpoints: {missing}") | |
| health = client.get("/health") | |
| tasks = client.get("/tasks") | |
| if health.status_code != 200 or tasks.status_code != 200: | |
| return CheckResult("OpenEnv endpoint contract", False, "Health/tasks endpoints not responding with 200") | |
| return CheckResult("OpenEnv endpoint contract", True, "Required endpoints are present and reachable") | |
| def check_tasks_and_graders() -> CheckResult: | |
| counts = {task_id: len(list_scenarios_for_task(task_id)) for task_id in (1, 2, 3)} | |
| if any(v < 1 for v in counts.values()): | |
| return CheckResult("3+ tasks with graders", False, f"Missing scenarios for tasks: {counts}") | |
| if len(counts) < 3: | |
| return CheckResult("3+ tasks with graders", False, "Expected at least 3 tasks") | |
| return CheckResult("3+ tasks with graders", True, f"Task scenarios present: {counts}") | |
| def check_phase1_api_flow() -> CheckResult: | |
| client = TestClient(app) | |
| reset = client.post("/reset", json={"task_id": 1, "seed": 42}) | |
| if reset.status_code != 200: | |
| return CheckResult("Baseline API flow", False, f"/reset failed with {reset.status_code}") | |
| session_id = reset.json()["session_id"] | |
| step = client.post( | |
| "/step", | |
| json={"action": {"action_type": "classify_alert", "severity": "P2", "service_name": "payment-service", "pattern_type": "database_overload"}}, | |
| headers={"X-Session-ID": session_id}, | |
| ) | |
| if step.status_code != 200: | |
| return CheckResult("Baseline API flow", False, f"/step failed with {step.status_code}") | |
| grade = client.post("/grader", json={"session_id": session_id}) | |
| if grade.status_code != 200: | |
| return CheckResult("Baseline API flow", False, f"/grader failed with {grade.status_code}") | |
| return CheckResult("Baseline API flow", True, "reset -> step -> grader path works") | |
| def check_grader_bounds() -> CheckResult: | |
| client = TestClient(app) | |
| scores: dict[int, float] = {} | |
| for task_id in (1, 2, 3): | |
| reset = client.post("/reset", json={"task_id": task_id, "seed": 42}) | |
| if reset.status_code != 200: | |
| return CheckResult("Grader bounds [0.0,1.0]", False, f"/reset failed for task {task_id}") | |
| session_id = reset.json()["session_id"] | |
| headers = {"X-Session-ID": session_id} | |
| done = False | |
| max_loops = 40 | |
| loops = 0 | |
| while not done and loops < max_loops: | |
| loops += 1 | |
| if task_id == 1: | |
| action = { | |
| "action_type": "classify_alert", | |
| "severity": "P2", | |
| "service_name": "payment-service", | |
| "pattern_type": "database_overload", | |
| } | |
| elif task_id == 2: | |
| if loops == 1: | |
| action = {"action_type": "filter_logs", "log_service": "checkout-service"} | |
| elif loops == 2: | |
| action = {"action_type": "get_metric", "metric_name": "error_rate", "service_name": "checkout-service"} | |
| elif loops == 3: | |
| action = {"action_type": "identify_service", "service_name": "checkout-service"} | |
| else: | |
| action = {"action_type": "propose_mitigation", "command": "kubectl rollout undo deployment/checkout-service"} | |
| else: | |
| state = client.get("/state", headers=headers) | |
| if state.status_code != 200: | |
| return CheckResult("Grader bounds [0.0,1.0]", False, f"/state failed for task {task_id}") | |
| state_data = state.json() | |
| if state_data.get("is_done"): | |
| done = True | |
| break | |
| if loops >= 20: | |
| action = { | |
| "action_type": "write_postmortem", | |
| "postmortem_text": "database connection pool checkout-service mitigation and prevention", | |
| } | |
| else: | |
| action = {"action_type": "no_op"} | |
| step = client.post("/step", json={"action": action}, headers=headers) | |
| if step.status_code != 200: | |
| return CheckResult("Grader bounds [0.0,1.0]", False, f"/step failed for task {task_id}: {step.status_code}") | |
| done = bool(step.json().get("done")) | |
| grade = client.post("/grader", json={"session_id": session_id}) | |
| if grade.status_code != 200: | |
| return CheckResult("Grader bounds [0.0,1.0]", False, f"/grader failed for task {task_id}") | |
| score = float(grade.json()["score"]) | |
| scores[task_id] = score | |
| if score < 0.0 or score > 1.0: | |
| return CheckResult("Grader bounds [0.0,1.0]", False, f"Task {task_id} score out of range: {score}") | |
| return CheckResult("Grader bounds [0.0,1.0]", True, f"Scores within range: {scores}") | |
| def check_baseline_reproducibility(runs: int) -> CheckResult: | |
| original_provider = os.environ.get("BASELINE_PROVIDER") | |
| original_url = os.environ.get("BASELINE_ENV_URL") | |
| os.environ["BASELINE_PROVIDER"] = "scripted" | |
| os.environ["BASELINE_ENV_URL"] = "asgi://local" | |
| try: | |
| results = [run_baseline_sync() for _ in range(runs)] | |
| finally: | |
| if original_provider is None: | |
| os.environ.pop("BASELINE_PROVIDER", None) | |
| else: | |
| os.environ["BASELINE_PROVIDER"] = original_provider | |
| if original_url is None: | |
| os.environ.pop("BASELINE_ENV_URL", None) | |
| else: | |
| os.environ["BASELINE_ENV_URL"] = original_url | |
| avgs = [r["average_score"] for r in results] | |
| unique = sorted(set(avgs)) | |
| passed = len(unique) == 1 | |
| return CheckResult( | |
| "Baseline reproduces", | |
| passed, | |
| f"Average scores across {runs} scripted runs: {avgs}", | |
| ) | |
| def check_docker_build() -> CheckResult: | |
| command = ["docker", "build", "-f", "Dockerfile", "-t", "incident-ops-env:readiness", "."] | |
| proc = subprocess.run(command, capture_output=True, text=True) | |
| if proc.returncode == 0: | |
| return CheckResult("Dockerfile builds", True, "docker build succeeded") | |
| text = (proc.stderr or proc.stdout).strip() | |
| if "permission denied" in text.lower() and "docker.sock" in text.lower(): | |
| return CheckResult("Dockerfile builds", False, "Blocked by Docker socket permissions on this machine") | |
| return CheckResult("Dockerfile builds", False, text.splitlines()[-1] if text else "docker build failed") | |
| def check_space_url(space_url: str) -> CheckResult: | |
| base = space_url.rstrip("/") | |
| try: | |
| with httpx.Client(timeout=30.0) as client: | |
| health = client.get(f"{base}/health") | |
| if health.status_code != 200: | |
| return CheckResult("Space URL smoke test", False, f"/health returned {health.status_code}") | |
| reset = client.post(f"{base}/reset", json={"task_id": 1, "seed": 42}) | |
| if reset.status_code != 200: | |
| return CheckResult("Space URL smoke test", False, f"/reset returned {reset.status_code}") | |
| except Exception as exc: | |
| return CheckResult("Space URL smoke test", False, f"Request failure: {exc}") | |
| return CheckResult("Space URL smoke test", True, "Space responded with 200 for /health and /reset") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="Phase readiness checks for IncidentOpsEnv.") | |
| parser.add_argument("--runs", type=int, default=3, help="Number of baseline runs for variance check.") | |
| parser.add_argument("--skip-docker", action="store_true", help="Skip Docker build gate check.") | |
| parser.add_argument("--space-url", type=str, default="", help="Optional deployed HF Space base URL for smoke checks.") | |
| args = parser.parse_args() | |
| checks = [ | |
| check_openenv_contract(), | |
| check_tasks_and_graders(), | |
| check_phase1_api_flow(), | |
| check_grader_bounds(), | |
| check_baseline_reproducibility(runs=args.runs), | |
| ] | |
| if not args.skip_docker: | |
| checks.append(check_docker_build()) | |
| if args.space_url: | |
| checks.append(check_space_url(args.space_url)) | |
| print("=== Phase 1: Automated Validation ===") | |
| for check in checks: | |
| state = "PASS" if check.passed else "FAIL" | |
| print(f"[{state}] {check.name}: {check.detail}") | |
| phase1_pass = all(c.passed for c in checks) | |
| print(f"Phase 1 gate: {'PASS' if phase1_pass else 'FAIL'}") | |
| print("\n=== Phase 2: Agentic Evaluation Readiness ===") | |
| print("- Scripted baseline reproducibility check included above.") | |
| print("- Run LLM baseline with provider keys set, e.g. OPENAI_API_KEY or GEMINI_API_KEY.") | |
| print("- For Nemotron/Open LLM runs, point your runner at /reset,/step,/grader and log per-task scores.") | |
| print("- Variance check recommendation: >=3 runs/model, compare mean and stddev.") | |
| print("\n=== Phase 3: Human Review Readiness ===") | |
| print("- Include README + docs/USAGE_AND_WORKING.md for setup clarity.") | |
| print("- Keep deterministic seeds and scenario ids in reports.") | |
| print("- Keep exploit notes: runbook leaks and action-schema edge cases should be disclosed.") | |
| print("\nMachine-readable:") | |
| print(json.dumps({"phase1_pass": phase1_pass, "checks": [c.__dict__ for c in checks]}, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |