Upload folder using huggingface_hub

README.md

@@ -114,15 +114,17 @@ Failure space: 4 services x 3 failure types x 2 severities = 24 distinct failure
 
 ## Reward Design
 
-Dense per-step reward that creates a learnable gradient for RL training. Investigation rewards are **information-gain aware** — viewing logs on a degraded service gives +0.04 (high info-gain) while a healthy service gives +0.01 (low info-gain). Health improvements give proportional reward via system health delta (+0.005 per 1% improvement). Breaking healthy services is heavily penalized (-0.30). Repeated identical actions are penalized (-0.02). All grading is outcome-based — no procedure-based criteria. Rewards are **task-adaptive** — harder tasks with time pressure get steeper gradients (1.0x–1.5x urgency scaling), creating a curriculum-aware reward landscape. Rewards are bounded [-0.35, +0.20] per step to prevent training instability.
+Dense per-step reward that creates a learnable gradient for RL training. Investigation rewards use **diminishing-returns exploration** — first investigation of an unhealthy service gives +0.04, with decay as more services are investigated. Health improvements give proportional reward via system health delta (+0.005 per 1% improvement). **Sub-goal milestones** reward intermediate progress: config fixes (+0.08), migrations (+0.06), and alert resolution (+0.03). Breaking healthy services is heavily penalized (-0.30). All grading is outcome-based — no procedure-based criteria. Rewards are **task-adaptive** — harder tasks with time pressure get steeper gradients (1.0x–1.5x urgency scaling), creating a curriculum-aware reward landscape. Rewards are bounded [-0.35, +0.20] per step to prevent training instability.
 
 | Signal | Reward | Condition |
 |--------|--------|-----------|
 | Service deployed to production | +0.15 | Service reaches prod successfully |
 | Service verified in staging | +0.05 | Staging health check passes |
-| Investigation (degraded svc) | +0.04 | First-time view on unhealthy service |
-| Investigation (healthy svc) | +0.01 | First-time view on healthy service |
-| Investigation (global) | +0.02 | First-time view_pipeline |
+| Config error fixed | +0.08 | Service health improved after config change |
+| Migration completed | +0.06 | Pending migration count decreased |
+| Alert resolved | +0.03 | Alert count decreased |
+| Investigation (degraded svc) | +0.04× | First-time view on unhealthy service (with decay) |
+| Investigation (healthy svc) | +0.01× | First-time view on healthy service (with decay) |
 | Health improvement | +0.005/1% | System health delta |
 | Broke healthy service | -0.30 | Service went from healthy to degraded/down |
 | Repeated investigation | -0.01 | Same view action on same target |

inference.py

@@ -4,6 +4,7 @@ import asyncio
 import json
 import os
 import textwrap
+from typing import List, Optional
 
 from openai import OpenAI
 
@@ -33,7 +34,7 @@ def log_start(task: str, env: str, model: str) -> None:
     print(f"[START] task={task} env={env} model={model}", flush=True)
 
 
-def log_step(step: int, action: str, reward: float, done: bool, error: str | None) -> None:
+def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
     error_val = error if error else "null"
     done_val = str(done).lower()
     print(
@@ -43,7 +44,7 @@ def log_step(step: int, action: str, reward: float, done: bool, error: str | Non
     )
 
 
-def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None:
+def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
     rewards_str = ",".join(f"{r:.2f}" for r in rewards)
     print(
         f"[END] success={str(success).lower()} steps={steps} "

requirements.txt

@@ -1,4 +1,5 @@
-openenv-core>=0.2.1
+openenv-core[core]>=0.2.2
 pydantic>=2.0
 fastapi>=0.104.0
 uvicorn>=0.24.0
+openai>=1.0.0

server/__init__.py

@@ -5,7 +5,3 @@
 # LICENSE file in the root directory of this source tree.
 
 """DevOps Pipeline environment server components."""
-
-from server.pipeline_environment import PipelineEnvironment
-
-__all__ = ["PipelineEnvironment"]

server/app.py

@@ -106,6 +106,9 @@ async def run_grader(task_name: str = "clean_deploy"):
         return {"task": task_name, "score": 0.0, "error": "No active session. Call /reset first."}
     if not env.get_episode_history():
         return {"task": env.get_task_name(), "score": 0.0, "error": "No steps taken. Call /step first."}
+    active_task = env.get_task_name()
+    if task_name and task_name != "clean_deploy" and task_name != active_task:
+        return {"task": task_name, "score": 0.0, "error": f"Task mismatch: active session is '{active_task}', not '{task_name}'."}
     score = _grade_task(
         env.get_task_name(),
         env.get_episode_history(),

server/graders.py

@@ -133,12 +133,18 @@ def grade_judgment_call(episode_history, engine):
         elif api_gw.latency_ms < 500:
             score += 0.10  # Partial improvement
 
-    # web-frontend collateral damage
+    # web-frontend collateral damage (smooth gradient)
     if web_fe:
-        if web_fe.health.value == "healthy" and web_fe.error_rate < 2.0:
+        if web_fe.health.value == "healthy" and web_fe.error_rate < 0.5:
             score += 0.25
+        elif web_fe.health.value == "healthy" and web_fe.error_rate < 2.0:
+            score += 0.20
+        elif web_fe.health.value == "healthy" and web_fe.error_rate < 5.0:
+            score += 0.15
         elif web_fe.error_rate < 10.0:
             score += 0.10
+        elif web_fe.error_rate < 20.0:
+            score += 0.05
 
     # Time to resolution — when api-gateway actually became healthy (outcome-based)
     resolution_step = len(episode_history)
@@ -255,8 +261,12 @@ def grade_capacity_crisis(episode_history, engine):
     # Root cause: database protected (0.30)
     if db:
         max_conn = int(db.config.get("max_connections", "50"))
-        if max_conn >= 100 and db.cpu_percent < 85:
-            score += 0.30
+        shared_buf = db.config.get("shared_buffers", "4GB")
+        shared_gb = int(shared_buf.replace("GB", "")) if "GB" in str(shared_buf) else 4
+        if max_conn >= 100 and db.cpu_percent < 85 and shared_gb >= 6:
+            score += 0.30  # Both configs optimized
+        elif max_conn >= 100 and db.cpu_percent < 85:
+            score += 0.25  # Connections fixed, buffers not
         elif max_conn >= 75 and db.cpu_percent < 85:
             score += 0.20
         elif max_conn >= 75:
@@ -320,7 +330,11 @@ def grade_random_incident(episode_history, engine):
         if not scenario.check_config_error(failing_name, failing_svc.config):
             score += 0.20
         elif getattr(scenario, '_failure_type', '') == 'degraded_performance':
-            score += 0.20  # No config error to fix for degraded_performance
+            # No config error — check if agent actually improved the service
+            if failing_svc.error_rate < 5.0 and failing_svc.health.value == "healthy":
+                score += 0.20  # Fully recovered
+            elif failing_svc.error_rate < 10.0:
+                score += 0.10  # Partially improved
 
     # No collateral damage (0.10) — outcome-based, not procedure-based
     any_broke = any(entry.get("broke_healthy", False) for entry in episode_history)

server/pipeline_engine.py

@@ -620,11 +620,14 @@ class PipelineEngine:
                     "latency_ms": s.latency_ms,
                     "prod_deployed": s.prod_deployed,
                     "staging_verified": s.staging_verified,
+                    "config": dict(s.config),
                 }
                 for name, s in self.services.items()
             },
             "system_health": self.get_system_health(),
             "pipeline_stage": self.pipeline_stage.value,
+            "migrations_pending": list(self.migrations_pending),
+            "alerts": list(self.alerts),
         }
 
     def get_system_health(self):

server/pipeline_environment.py

@@ -31,7 +31,7 @@ TASK_SEEDS = {
     "judgment_call": 3003,
     "cascading_failure": 4004,
     "capacity_crisis": 5005,
-    "random_incident": 6006,
+    "random_incident": int(os.environ.get("DEVOPS_SEED", "6006")),
 }
 
 TASK_MAX_STEPS = {
@@ -232,6 +232,19 @@ class PipelineEnvironment(Environment):
                 )
             elif svc_state.cpu_percent > 80:
                 alerts.append(f"CAUTION: {name} CPU high ({svc_state.cpu_percent:.0f}%)")
+        # Add dependency chain hints for degraded services
+        for name, svc_state in self._engine.services.items():
+            if svc_state.health in (ServiceHealth.DEGRADED, ServiceHealth.DOWN):
+                upstream_issues = [
+                    d for d in svc_state.dependencies
+                    if d in self._engine.services
+                    and self._engine.services[d].health in (ServiceHealth.DEGRADED, ServiceHealth.DOWN)
+                ]
+                if upstream_issues:
+                    alerts.append(
+                        f"HINT: {name} depends on {', '.join(upstream_issues)} "
+                        f"(also unhealthy — root cause likely upstream)"
+                    )
         summary = "; ".join(alerts) if alerts else "All services nominal."
 
         return PipelineObservation(

server/rewards.py

@@ -23,58 +23,77 @@ TASK_URGENCY = {
 def calculate_reward(prev_snapshot, current_snapshot, action, viewed_actions,
                      last_action_key=None, task_name="clean_deploy"):
     """
-    Outcome-based reward. No procedure bonuses.
-    viewed_actions: a set tracking first-time investigation keys (e.g. "view_logs:api-gateway")
-    last_action_key: string key of previous action for repeat detection
-    task_name: current task for urgency scaling
+    Outcome-based reward. No procedure bonuses. No early returns.
+    ALL actions go through the full reward pipeline.
     Returns a float bounded to [-0.35, +0.20].
     """
     reward = 0.0
 
-    # 1. System health delta (primary signal)
+    # 1. System health delta (ALL actions)
     health_delta = current_snapshot["system_health"] - prev_snapshot["system_health"]
-    reward += health_delta * 0.005  # +0.005 per 1% health improvement
+    reward += health_delta * 0.005
 
-    # 2. Deployment progress
+    # 2. Deployment progress (ALL actions — recovery ticks can change state)
     for name, curr_svc in current_snapshot["services"].items():
         prev_svc = prev_snapshot["services"].get(name, {})
         if curr_svc["prod_deployed"] and not prev_svc.get("prod_deployed"):
-            reward += 0.15  # Service successfully reached production
+            reward += 0.15
         if curr_svc["staging_verified"] and not prev_svc.get("staging_verified"):
-            reward += 0.05  # Service verified in staging
+            reward += 0.05
 
-    # 3. Broke something that was working
+    # 3. Broke healthy service penalty (ALL actions)
     for name, curr_svc in current_snapshot["services"].items():
         prev_svc = prev_snapshot["services"].get(name, {})
         if prev_svc.get("health") == "healthy" and curr_svc["health"] in ("degraded", "down"):
-            reward -= 0.30  # Catastrophic penalty
+            reward -= 0.30
 
-    # 4. Investigation bonus (info-gain aware)
+    # 4. Sub-goal completion rewards (intermediate milestones for RL)
+    # Config change detected — check if it fixed an error
+    for name, curr_svc in current_snapshot["services"].items():
+        prev_svc = prev_snapshot["services"].get(name, {})
+        prev_config = prev_svc.get("config", {})
+        curr_config = curr_svc.get("config", {})
+        if prev_config and curr_config and prev_config != curr_config:
+            # Config changed — reward if health improved on this service
+            if prev_svc.get("health") in ("degraded", "down") and curr_svc["health"] == "healthy":
+                reward += 0.08
+
+    # Migration completed
+    prev_pending = len(prev_snapshot.get("migrations_pending", []))
+    curr_pending = len(current_snapshot.get("migrations_pending", []))
+    if curr_pending < prev_pending:
+        reward += 0.06
+
+    # Alert resolved
+    prev_alerts = len(prev_snapshot.get("alerts", []))
+    curr_alerts = len(current_snapshot.get("alerts", []))
+    if curr_alerts < prev_alerts:
+        reward += 0.03
+
+    # 5. Investigation bonus with diminishing returns (view_* actions only)
     if action.action_type in (ActionType.VIEW_PIPELINE, ActionType.VIEW_LOGS, ActionType.VIEW_CONFIG):
         action_key = f"{action.action_type.value}:{action.service_name or 'global'}"
         if action_key not in viewed_actions:
             viewed_actions.add(action_key)
-            # Higher reward for investigating unhealthy services (more info gain)
+            investigation_count = len(viewed_actions)
+            decay_factor = 1.0 / (1 + (investigation_count - 1) * 0.3)
             if action.service_name:
                 svc_data = current_snapshot["services"].get(action.service_name, {})
                 if svc_data.get("health") in ("degraded", "down"):
-                    reward += 0.04  # High info-gain
+                    reward += 0.04 * decay_factor
                 else:
-                    reward += 0.01  # Low info-gain (healthy service)
+                    reward += 0.01 * decay_factor
             else:
-                reward += 0.02  # view_pipeline (global)
+                reward += 0.02 * decay_factor
         else:
-            reward -= 0.01  # Repeated investigation of same target
-
-        # Apply urgency scaling and bound
-        reward *= TASK_URGENCY.get(task_name, 1.0)
-        return max(min(reward, 0.20), -0.35)
+            reward -= 0.01
 
-    # 5. Repeated exact action penalty
-    current_action_key = f"{action.action_type.value}:{action.service_name or ''}"
-    if last_action_key and current_action_key == last_action_key:
-        reward -= 0.02  # Exact repeat penalty
+    # 6. Repeated exact action penalty (non-view actions)
+    if action.action_type not in (ActionType.VIEW_PIPELINE, ActionType.VIEW_LOGS, ActionType.VIEW_CONFIG):
+        current_action_key = f"{action.action_type.value}:{action.service_name or ''}"
+        if last_action_key and current_action_key == last_action_key:
+            reward -= 0.02
 
-    # 6. Apply task urgency scaling and bound
+    # 7. Apply task urgency scaling and bound
     reward *= TASK_URGENCY.get(task_name, 1.0)
     return max(min(reward, 0.20), -0.35)