incident_ops_env / scripts /phase_readiness.py
rawknee-69's picture
here you go
9d7a42e
Raw History Blame Contribute Delete
10.1 kB
from __future__ import annotations
import argparse
import json
import os
import subprocess
from dataclasses import dataclass
import httpx
from fastapi.testclient import TestClient
from baseline import run_baseline_sync
from incident_ops_env.server.app import app
from incident_ops_env.server.scenario_loader import list_scenarios_for_task
@dataclass
class CheckResult:
name: str
passed: bool
detail: str
def check_openenv_contract() -> CheckResult:
client = TestClient(app)
required = ["/health", "/tasks", "/reset", "/step", "/state", "/grader", "/baseline"]
found = {route.path for route in app.routes}
missing = [path for path in required if path not in found]
if missing:
return CheckResult("OpenEnv endpoint contract", False, f"Missing endpoints: {missing}")
health = client.get("/health")
tasks = client.get("/tasks")
if health.status_code != 200 or tasks.status_code != 200:
return CheckResult("OpenEnv endpoint contract", False, "Health/tasks endpoints not responding with 200")
return CheckResult("OpenEnv endpoint contract", True, "Required endpoints are present and reachable")
def check_tasks_and_graders() -> CheckResult:
counts = {task_id: len(list_scenarios_for_task(task_id)) for task_id in (1, 2, 3)}
if any(v < 1 for v in counts.values()):
return CheckResult("3+ tasks with graders", False, f"Missing scenarios for tasks: {counts}")
if len(counts) < 3:
return CheckResult("3+ tasks with graders", False, "Expected at least 3 tasks")
return CheckResult("3+ tasks with graders", True, f"Task scenarios present: {counts}")
def check_phase1_api_flow() -> CheckResult:
client = TestClient(app)
reset = client.post("/reset", json={"task_id": 1, "seed": 42})
if reset.status_code != 200:
return CheckResult("Baseline API flow", False, f"/reset failed with {reset.status_code}")
session_id = reset.json()["session_id"]
step = client.post(
"/step",
json={"action": {"action_type": "classify_alert", "severity": "P2", "service_name": "payment-service", "pattern_type": "database_overload"}},
headers={"X-Session-ID": session_id},
)
if step.status_code != 200:
return CheckResult("Baseline API flow", False, f"/step failed with {step.status_code}")
grade = client.post("/grader", json={"session_id": session_id})
if grade.status_code != 200:
return CheckResult("Baseline API flow", False, f"/grader failed with {grade.status_code}")
return CheckResult("Baseline API flow", True, "reset -> step -> grader path works")
def check_grader_bounds() -> CheckResult:
client = TestClient(app)
scores: dict[int, float] = {}
for task_id in (1, 2, 3):
reset = client.post("/reset", json={"task_id": task_id, "seed": 42})
if reset.status_code != 200:
return CheckResult("Grader bounds [0.0,1.0]", False, f"/reset failed for task {task_id}")
session_id = reset.json()["session_id"]
headers = {"X-Session-ID": session_id}
done = False
max_loops = 40
loops = 0
while not done and loops < max_loops:
loops += 1
if task_id == 1:
action = {
"action_type": "classify_alert",
"severity": "P2",
"service_name": "payment-service",
"pattern_type": "database_overload",
}
elif task_id == 2:
if loops == 1:
action = {"action_type": "filter_logs", "log_service": "checkout-service"}
elif loops == 2:
action = {"action_type": "get_metric", "metric_name": "error_rate", "service_name": "checkout-service"}
elif loops == 3:
action = {"action_type": "identify_service", "service_name": "checkout-service"}
else:
action = {"action_type": "propose_mitigation", "command": "kubectl rollout undo deployment/checkout-service"}
else:
state = client.get("/state", headers=headers)
if state.status_code != 200:
return CheckResult("Grader bounds [0.0,1.0]", False, f"/state failed for task {task_id}")
state_data = state.json()
if state_data.get("is_done"):
done = True
break
if loops >= 20:
action = {
"action_type": "write_postmortem",
"postmortem_text": "database connection pool checkout-service mitigation and prevention",
}
else:
action = {"action_type": "no_op"}
step = client.post("/step", json={"action": action}, headers=headers)
if step.status_code != 200:
return CheckResult("Grader bounds [0.0,1.0]", False, f"/step failed for task {task_id}: {step.status_code}")
done = bool(step.json().get("done"))
grade = client.post("/grader", json={"session_id": session_id})
if grade.status_code != 200:
return CheckResult("Grader bounds [0.0,1.0]", False, f"/grader failed for task {task_id}")
score = float(grade.json()["score"])
scores[task_id] = score
if score < 0.0 or score > 1.0:
return CheckResult("Grader bounds [0.0,1.0]", False, f"Task {task_id} score out of range: {score}")
return CheckResult("Grader bounds [0.0,1.0]", True, f"Scores within range: {scores}")
def check_baseline_reproducibility(runs: int) -> CheckResult:
original_provider = os.environ.get("BASELINE_PROVIDER")
original_url = os.environ.get("BASELINE_ENV_URL")
os.environ["BASELINE_PROVIDER"] = "scripted"
os.environ["BASELINE_ENV_URL"] = "asgi://local"
try:
results = [run_baseline_sync() for _ in range(runs)]
finally:
if original_provider is None:
os.environ.pop("BASELINE_PROVIDER", None)
else:
os.environ["BASELINE_PROVIDER"] = original_provider
if original_url is None:
os.environ.pop("BASELINE_ENV_URL", None)
else:
os.environ["BASELINE_ENV_URL"] = original_url
avgs = [r["average_score"] for r in results]
unique = sorted(set(avgs))
passed = len(unique) == 1
return CheckResult(
"Baseline reproduces",
passed,
f"Average scores across {runs} scripted runs: {avgs}",
)
def check_docker_build() -> CheckResult:
command = ["docker", "build", "-f", "Dockerfile", "-t", "incident-ops-env:readiness", "."]
proc = subprocess.run(command, capture_output=True, text=True)
if proc.returncode == 0:
return CheckResult("Dockerfile builds", True, "docker build succeeded")
text = (proc.stderr or proc.stdout).strip()
if "permission denied" in text.lower() and "docker.sock" in text.lower():
return CheckResult("Dockerfile builds", False, "Blocked by Docker socket permissions on this machine")
return CheckResult("Dockerfile builds", False, text.splitlines()[-1] if text else "docker build failed")
def check_space_url(space_url: str) -> CheckResult:
base = space_url.rstrip("/")
try:
with httpx.Client(timeout=30.0) as client:
health = client.get(f"{base}/health")
if health.status_code != 200:
return CheckResult("Space URL smoke test", False, f"/health returned {health.status_code}")
reset = client.post(f"{base}/reset", json={"task_id": 1, "seed": 42})
if reset.status_code != 200:
return CheckResult("Space URL smoke test", False, f"/reset returned {reset.status_code}")
except Exception as exc:
return CheckResult("Space URL smoke test", False, f"Request failure: {exc}")
return CheckResult("Space URL smoke test", True, "Space responded with 200 for /health and /reset")
def main() -> None:
parser = argparse.ArgumentParser(description="Phase readiness checks for IncidentOpsEnv.")
parser.add_argument("--runs", type=int, default=3, help="Number of baseline runs for variance check.")
parser.add_argument("--skip-docker", action="store_true", help="Skip Docker build gate check.")
parser.add_argument("--space-url", type=str, default="", help="Optional deployed HF Space base URL for smoke checks.")
args = parser.parse_args()
checks = [
check_openenv_contract(),
check_tasks_and_graders(),
check_phase1_api_flow(),
check_grader_bounds(),
check_baseline_reproducibility(runs=args.runs),
]
if not args.skip_docker:
checks.append(check_docker_build())
if args.space_url:
checks.append(check_space_url(args.space_url))
print("=== Phase 1: Automated Validation ===")
for check in checks:
state = "PASS" if check.passed else "FAIL"
print(f"[{state}] {check.name}: {check.detail}")
phase1_pass = all(c.passed for c in checks)
print(f"Phase 1 gate: {'PASS' if phase1_pass else 'FAIL'}")
print("\n=== Phase 2: Agentic Evaluation Readiness ===")
print("- Scripted baseline reproducibility check included above.")
print("- Run LLM baseline with provider keys set, e.g. OPENAI_API_KEY or GEMINI_API_KEY.")
print("- For Nemotron/Open LLM runs, point your runner at /reset,/step,/grader and log per-task scores.")
print("- Variance check recommendation: >=3 runs/model, compare mean and stddev.")
print("\n=== Phase 3: Human Review Readiness ===")
print("- Include README + docs/USAGE_AND_WORKING.md for setup clarity.")
print("- Keep deterministic seeds and scenario ids in reports.")
print("- Keep exploit notes: runbook leaks and action-schema edge cases should be disclosed.")
print("\nMachine-readable:")
print(json.dumps({"phase1_pass": phase1_pass, "checks": [c.__dict__ for c in checks]}, indent=2))
if __name__ == "__main__":
main()