Add inference.py, stress_test.py, check.py 1000-sim validated

check.py

@@ -0,0 +1,49 @@
+"""Quick smoke test: checks all 3 required items for repair_env_v2."""
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+root = os.path.dirname(os.path.abspath(__file__))
+
+print("=" * 50)
+print("  repair_env_v2 — Pre-flight Check")
+print("=" * 50)
+
+# --- 1. Dockerfile ---
+df = os.path.join(root, "Dockerfile")
+ok1 = os.path.isfile(df)
+print(f"[{'OK' if ok1 else 'MISSING'}] Dockerfile")
+
+# --- 2. inference.py ---
+inf = os.path.join(root, "inference.py")
+ok2 = os.path.isfile(inf)
+print(f"[{'OK' if ok2 else 'MISSING'}] inference.py")
+
+# --- 3. openenv.yaml ---
+yaml_f = os.path.join(root, "openenv.yaml")
+ok3 = os.path.isfile(yaml_f)
+print(f"[{'OK' if ok3 else 'MISSING'}] openenv.yaml")
+
+# --- 4. inference.py actually runs ---
+print("\n--- Running inference on all 4 tasks (verbose=False) ---")
+try:
+    import inference as inf_mod
+    from tasks_v2 import TASKS
+    results = []
+    for tid in TASKS:
+        r = inf_mod.run_inference(task_id=tid, verbose=False)
+        results.append(r)
+
+    print(f"\n{'Task':<12} {'Steps':>5} {'Optimal':>7}  {'Reward':>7}  {'Score':>6}  {'Success':>7}  {'Health':>6}")
+    print("-" * 60)
+    for r in results:
+        opt = TASKS[r["task_id"]]["optimal_steps"]
+        print(f"{r['task_id']:<12} {r['steps_taken']:>5} {opt:>7}  {r['total_reward']:>7.3f}  {r['score']:>6.3f}  {str(r['success']):>7}  {r['final_health_ratio']:>6.2%}")
+    print("\n[OK] inference.py runs cleanly on all tasks")
+except Exception as e:
+    print(f"[FAIL] inference.py error: {e}")
+    import traceback; traceback.print_exc()
+
+print("\n" + "=" * 50)
+all_ok = ok1 and ok2 and ok3
+print("RESULT:", "ALL CHECKS PASSED" if all_ok else "SOME CHECKS FAILED")
+print("=" * 50)

inference.py

@@ -0,0 +1,184 @@
+"""
+inference.py — OpenEnv standard inference entry point for Repair Strategy System V2.
+
+Demonstrates how an external agent interacts with RepairEnvV2:
+  1. Pick a task
+  2. Reset the environment with that task config
+  3. Step through with actions
+  4. Grade the final episode via get_episode_result()
+
+Run:
+    python inference.py                        # runs 'easy' task
+    python inference.py --task medium
+    python inference.py --task hard
+    python inference.py --task multi_root
+    python inference.py --all                  # runs all 4 tasks and prints a summary table
+"""
+
+from __future__ import annotations
+
+import sys
+import os
+import argparse
+
+# Fix Windows console encoding for non-ASCII characters
+if sys.stdout.encoding and sys.stdout.encoding.lower() != 'utf-8':
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+
+# Ensure the repo root is on the path regardless of where this is called from
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from env_v2 import RepairEnvV2
+from models_v2 import Action, ActionType, ComponentTarget
+from tasks_v2 import TASKS
+
+
+# ---------------------------------------------------------------------------
+# Simple deterministic baseline agent (inspect root-cause area, then repair)
+# ---------------------------------------------------------------------------
+
+def inspect_then_fix_agent(obs, task_config: dict, step: int) -> Action:
+    """
+    Minimal deterministic agent:
+    - Step 0: inspect the (first) suspected root cause component
+    - Step 1+: attempt repair on whichever component has the highest error_rate
+    """
+    root_cause = task_config["root_cause"]
+    if isinstance(root_cause, list):
+        root_cause = root_cause[0]
+
+    # Step 0: inspect first
+    if step == 0:
+        inspect_map = {
+            "database": ActionType.inspect_database,
+            "api":      ActionType.inspect_api,
+            "cache":    ActionType.inspect_cache,
+            "queue":    ActionType.inspect_queue,
+        }
+        return Action(
+            action_type=inspect_map.get(root_cause, ActionType.inspect_database),
+            target=ComponentTarget(root_cause),
+        )
+
+    # Step 1+: pick component with worst error_rate and try to fix it
+    worst_comp = max(obs.metrics.items(), key=lambda kv: kv[1].error_rate)[0]
+    repair_map = {
+        "database": ActionType.repair_database,
+        "api":      ActionType.restart_service,
+        "cache":    ActionType.clear_cache,
+        "queue":    ActionType.restart_service,
+    }
+    return Action(
+        action_type=repair_map.get(worst_comp, ActionType.restart_service),
+        target=ComponentTarget(worst_comp),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Main inference loop
+# ---------------------------------------------------------------------------
+
+def run_inference(task_id: str = "easy", verbose: bool = True) -> dict:
+    """
+    Run one full episode on the specified task using the inspect-then-fix agent.
+
+    Returns the EpisodeResultV2 as a dict with:
+        task_id, steps_taken, total_reward, score, success,
+        final_health_ratio, efficiency, correct_root_fix_ratio
+    """
+    if task_id not in TASKS:
+        raise ValueError(
+            f"Unknown task_id '{task_id}'. Choose from: {list(TASKS.keys())}"
+        )
+
+    config = TASKS[task_id]
+
+    # --- V2 API: __init__() takes no args; reset() takes (task_config, task_id) ---
+    env = RepairEnvV2()
+    obs = env.reset(task_config=config, task_id=task_id)
+
+    done = False
+    step = 0
+
+    if verbose:
+        print(f"\n{'='*62}")
+        print(f"  Task       : {task_id}")
+        print(f"  Difficulty : {config.get('difficulty', task_id)}")
+        print(f"  Max steps  : {config['max_steps']}  |  Optimal: {config['optimal_steps']}")
+        print(f"  Description: {config['description']}")
+        print(f"{'='*62}")
+
+    while not done:
+        action = inspect_then_fix_agent(obs, config, step)
+        obs, reward, done, info = env.step(action)
+
+        if verbose:
+            print(
+                f"  Step {step + 1:>2}: {str(action.action_type):<22} > {str(action.target):<10}"
+                f"| reward={reward.value:+.2f}  reason={reward.reason}"
+            )
+        step += 1
+
+    # --- Use the env's built-in result calculation ---
+    result = env.get_episode_result()
+
+    if verbose:
+        print(f"\n  -- Episode complete ------------------------------------------")
+        print(f"  Steps taken        : {result.steps_taken} (optimal: {config['optimal_steps']})")
+        print(f"  Total reward       : {result.total_reward:.4f}")
+        print(f"  Score              : {result.score:.4f}")
+        print(f"  Success            : {result.success}")
+        print(f"  Final health ratio : {result.final_health_ratio:.2%}")
+        print(f"  Efficiency         : {result.efficiency:.2%}")
+        print(f"  Correct root fixes : {result.correct_root_fix_ratio:.2%}")
+        print(f"{'='*62}\n")
+
+    return result.model_dump()
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="OpenEnv inference runner — Repair Strategy System V2"
+    )
+    parser.add_argument(
+        "--task",
+        default="easy",
+        choices=list(TASKS.keys()),
+        help="Task ID to run (default: easy)",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Run ALL tasks and print a summary table",
+    )
+    args = parser.parse_args()
+
+    if args.all:
+        print("\nRunning all tasks...\n")
+        results = []
+        for tid in TASKS:
+            r = run_inference(task_id=tid, verbose=True)
+            results.append(r)
+
+        # Summary table
+        print(f"\n{'-'*70}")
+        print(f"{'Task':<12} {'Steps':>6} {'Optimal':>8} {'Reward':>8} {'Score':>7} {'Success':>8} {'Health':>7}")
+        print(f"{'-'*70}")
+        for r in results:
+            optimal = TASKS[r['task_id']]['optimal_steps']
+            print(
+                f"{r['task_id']:<12} {r['steps_taken']:>6} {optimal:>8} "
+                f"{r['total_reward']:>8.3f} {r['score']:>7.3f} "
+                f"{str(r['success']):>8} {r['final_health_ratio']:>7.2%}"
+            )
+        print(f"{'-'*70}\n")
+    else:
+        run_inference(task_id=args.task, verbose=True)
+
+
+if __name__ == "__main__":
+    main()

stress_test.py

@@ -0,0 +1,192 @@
+"""
+stress_test.py — Run 1000 simulations across all tasks with varying seeds.
+Collects stats: success rate, avg score, avg reward, avg steps, failures.
+"""
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from env_v2 import RepairEnvV2
+from models_v2 import Action, ActionType, ComponentTarget
+from tasks_v2 import TASKS
+
+import random
+import time
+from collections import defaultdict
+
+# ---------------------------------------------------------------------------
+# Reuse the same inspect-then-fix agent from inference.py
+# ---------------------------------------------------------------------------
+
+def inspect_then_fix_agent(obs, task_config: dict, step: int, rng: random.Random) -> Action:
+    root_cause = task_config["root_cause"]
+    if isinstance(root_cause, list):
+        root_cause = root_cause[0]
+
+    if step == 0:
+        inspect_map = {
+            "database": ActionType.inspect_database,
+            "api":      ActionType.inspect_api,
+            "cache":    ActionType.inspect_cache,
+            "queue":    ActionType.inspect_queue,
+        }
+        return Action(
+            action_type=inspect_map.get(root_cause, ActionType.inspect_database),
+            target=ComponentTarget(root_cause),
+        )
+
+    worst_comp = max(obs.metrics.items(), key=lambda kv: kv[1].error_rate)[0]
+    repair_map = {
+        "database": ActionType.repair_database,
+        "api":      ActionType.restart_service,
+        "cache":    ActionType.clear_cache,
+        "queue":    ActionType.restart_service,
+    }
+    return Action(
+        action_type=repair_map.get(worst_comp, ActionType.restart_service),
+        target=ComponentTarget(worst_comp),
+    )
+
+
+def random_agent(obs, rng: random.Random) -> Action:
+    """Completely random agent for chaos testing."""
+    return Action(
+        action_type=rng.choice(list(ActionType)),
+        target=rng.choice(list(ComponentTarget)),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Run N simulations
+# ---------------------------------------------------------------------------
+
+def run_stress_test(n_total: int = 1000):
+    task_ids = list(TASKS.keys())
+    n_per_task = n_total // len(task_ids)
+
+    # Stats per task per agent
+    stats = {}
+    for task_id in task_ids:
+        stats[task_id] = {
+            "baseline": defaultdict(list),
+            "random":   defaultdict(list),
+        }
+
+    errors = []
+    total_runs = 0
+    start = time.time()
+
+    print(f"\n{'='*60}")
+    print(f"  Stress Test: {n_total} simulations")
+    print(f"  Tasks: {task_ids}")
+    print(f"  {n_per_task} runs per task  x  2 agents  =  {n_per_task * 2 * len(task_ids)} total episodes")
+    print(f"{'='*60}\n")
+
+    for task_id in task_ids:
+        config = TASKS[task_id]
+        print(f"  Running task '{task_id}' ({n_per_task} seeds x 2 agents)...", end="", flush=True)
+
+        for seed in range(n_per_task):
+            for agent_name in ["baseline", "random"]:
+                try:
+                    env = RepairEnvV2()
+                    obs = env.reset(task_config=config, task_id=task_id)
+
+                    done = False
+                    step = 0
+                    rng = random.Random(seed)
+
+                    while not done:
+                        if agent_name == "baseline":
+                            action = inspect_then_fix_agent(obs, config, step, rng)
+                        else:
+                            action = random_agent(obs, rng)
+
+                        obs, reward, done, info = env.step(action)
+                        step += 1
+
+                    result = env.get_episode_result()
+                    s = stats[task_id][agent_name]
+                    s["scores"].append(result.score)
+                    s["rewards"].append(result.total_reward)
+                    s["steps"].append(result.steps_taken)
+                    s["success"].append(int(result.success))
+                    s["health"].append(result.final_health_ratio)
+                    total_runs += 1
+
+                except Exception as e:
+                    errors.append({"task": task_id, "seed": seed, "agent": agent_name, "error": str(e)})
+
+        print(f" done.")
+
+    elapsed = time.time() - start
+
+    # ---------------------------------------------------------------------------
+    # Print results
+    # ---------------------------------------------------------------------------
+
+    print(f"\n{'='*60}")
+    print(f"  RESULTS  ({total_runs} episodes in {elapsed:.1f}s)")
+    print(f"{'='*60}\n")
+
+    for task_id in task_ids:
+        config = TASKS[task_id]
+        print(f"  Task: {task_id}  [{config.get('difficulty','?')}]  (optimal={config['optimal_steps']})")
+        print(f"  {'-'*55}")
+
+        for agent_name in ["baseline", "random"]:
+            s = stats[task_id][agent_name]
+            n = len(s["scores"])
+            if n == 0:
+                print(f"    {agent_name:<10}: NO DATA")
+                continue
+
+            avg_score   = sum(s["scores"])   / n
+            avg_reward  = sum(s["rewards"])  / n
+            avg_steps   = sum(s["steps"])    / n
+            success_pct = sum(s["success"])  / n * 100
+            avg_health  = sum(s["health"])   / n
+            min_score   = min(s["scores"])
+            max_score   = max(s["scores"])
+
+            print(f"    {agent_name:<10}:  n={n}  success={success_pct:5.1f}%  "
+                  f"score={avg_score:.3f} [{min_score:.2f}-{max_score:.2f}]  "
+                  f"reward={avg_reward:+.2f}  steps={avg_steps:.1f}  health={avg_health:.2%}")
+        print()
+
+    # Summary table
+    print(f"  {'Task':<12} {'Agent':<10} {'Success%':>9} {'AvgScore':>9} {'AvgReward':>10} {'AvgSteps':>9}")
+    print(f"  {'-'*58}")
+    for task_id in task_ids:
+        for agent_name in ["baseline", "random"]:
+            s = stats[task_id][agent_name]
+            n = len(s["scores"])
+            if n == 0: continue
+            print(f"  {task_id:<12} {agent_name:<10} "
+                  f"{sum(s['success'])/n*100:>8.1f}% "
+                  f"{sum(s['scores'])/n:>9.4f} "
+                  f"{sum(s['rewards'])/n:>10.3f} "
+                  f"{sum(s['steps'])/n:>9.1f}")
+    print(f"  {'-'*58}")
+
+    if errors:
+        print(f"\n  ERRORS ({len(errors)} total):")
+        for e in errors[:10]:
+            print(f"    [{e['task']} seed={e['seed']} {e['agent']}] {e['error']}")
+        if len(errors) > 10:
+            print(f"    ... and {len(errors) - 10} more")
+    else:
+        print(f"\n  No errors! All {total_runs} episodes completed cleanly.")
+
+    print(f"\n{'='*60}")
+    print(f"  Stress test complete. {total_runs}/{n_per_task * 2 * len(task_ids)} episodes ran successfully.")
+    print(f"{'='*60}\n")
+
+    return stats, errors
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--n", type=int, default=1000, help="Total simulations (split across tasks)")
+    args = parser.parse_args()
+    run_stress_test(n_total=args.n)