#!/usr/bin/env python3 """ OpenEnv Spec Compliance Validator for CodeReviewEnv v2 (MCP pattern) """ import sys import os import random import yaml from server.code_review_environment import CodeReviewEnvironment from models import CodeReviewAction, CodeReviewObservation, CodeReviewState from snippet_bank import SNIPPET_BANK, BUG_INJECTORS, generate_episode def check(name: str, condition: bool, reason: str = "") -> bool: status = "PASS" if condition else "FAIL" msg = f" [{status}] {name}" if not condition and reason: msg += f" -- {reason}" print(msg) return condition def _tool(tool_name, arguments=None): return CodeReviewAction(action_type="ToolCallAction", tool_name=tool_name, arguments=arguments or {}) def validate(): print("=" * 60) print("CodeReviewEnv -- OpenEnv Compliance Validation (MCP pattern)") print("=" * 60) results = [] # 1. reset() returns observation with tools_list print("\n--- Core Interface (MCP) ---") for difficulty in ["easy", "medium", "hard"]: try: env = CodeReviewEnvironment() obs = env.reset(seed=42, difficulty=difficulty) results.append(check( f"reset() returns tools_list ({difficulty})", isinstance(obs, CodeReviewObservation) and obs.tools_list is not None and len(obs.tools_list) == 8, )) except Exception as e: results.append(check(f"reset() ({difficulty})", False, str(e))) # 2. ListToolsAction works try: env = CodeReviewEnvironment() env.reset(seed=42) obs = env.step(CodeReviewAction(action_type="ListToolsAction")) tool_names = {t["name"] for t in obs.tools_list} results.append(check( "ListToolsAction returns 5 tools", tool_names == {"get_code", "run_code", "run_tests", "analyze_code", "check_line", "get_hint", "submit_fix", "submit_review"}, )) except Exception as e: results.append(check("ListToolsAction", False, str(e))) # 3. get_code returns code try: env = CodeReviewEnvironment() env.reset(seed=42) obs = env.step(_tool("get_code")) results.append(check( "get_code returns code", obs.success and obs.tool_result and len(obs.tool_result.get("code", "")) > 0, )) except Exception as e: results.append(check("get_code", False, str(e))) # 4. check_line returns reward try: env = CodeReviewEnvironment() env.reset(seed=42) state = env.state if state.gold_bugs: line = state.gold_bugs[0]["lines"][0] obs = env.step(_tool("check_line", {"line": line})) results.append(check( f"check_line({line}) gives positive reward", obs.success and obs.reward > 0, )) else: results.append(check("check_line (no bugs to test)", True)) except Exception as e: results.append(check("check_line", False, str(e))) # 5. submit_review ends episode try: env = CodeReviewEnvironment() env.reset(seed=42) obs = env.step(_tool("submit_review", {"issues": ["test"], "comment": "review"})) results.append(check( "submit_review ends episode (done=True)", obs.done is True and obs.reward is not None and 0.0 <= obs.reward <= 1.0, )) except Exception as e: results.append(check("submit_review", False, str(e))) # 6. state contains gold_bugs try: env = CodeReviewEnvironment() env.reset(seed=42) state = env.state results.append(check( "state contains gold_bugs", isinstance(state, CodeReviewState) and isinstance(state.gold_bugs, list), )) except Exception as e: results.append(check("state", False, str(e))) # 7. Reward in [0,1] across many seeds try: all_valid = True for seed in range(50): env = CodeReviewEnvironment() env.reset(seed=seed, difficulty=random.choice(["easy", "medium", "hard"])) obs = env.step(_tool("submit_review", {"issues": ["bug"], "comment": "test"})) if obs.reward < 0.0 or obs.reward > 1.0: all_valid = False break results.append(check("reward always in [0, 1]", all_valid)) except Exception as e: results.append(check("reward bounds", False, str(e))) # 8. Max steps forces submit try: env = CodeReviewEnvironment() env.reset(seed=42) for i in range(10): obs = env.step(_tool("analyze_code")) if obs.done: break results.append(check("max_steps forces submit", obs.done is True)) except Exception as e: results.append(check("max_steps", False, str(e))) # ── Procedural Generation ── print("\n--- Procedural Generation ---") try: codes = {generate_episode(seed=s, difficulty="easy")[1][:100] for s in range(10)} results.append(check(f"Different seeds -> different episodes ({len(codes)}/10)", len(codes) >= 3)) except Exception as e: results.append(check("different seeds", False, str(e))) try: _, c1, _ = generate_episode(seed=42) _, c2, _ = generate_episode(seed=42) results.append(check("seed=42 reproducibility", c1 == c2)) except Exception as e: results.append(check("reproducibility", False, str(e))) results.append(check(f"Snippet bank has {len(SNIPPET_BANK)} entries (>= 30)", len(SNIPPET_BANK) >= 30)) results.append(check(f"Bug injectors: {len(BUG_INJECTORS)} (>= 9)", len(BUG_INJECTORS) >= 9)) # ── Reward Quality ── print("\n--- Reward Quality ---") try: env = CodeReviewEnvironment() env.reset(seed=42) state = env.state if state.gold_bugs: g = state.gold_bugs[0] obs = env.step(_tool("submit_review", { "issues": [g["description"]], "flagged_lines": g["lines"], "suggestion": g["fix"], "comment": f"Found {g['bug_type']}. {g['fix']}", })) results.append(check(f"Perfect agent >= 0.5 (got {obs.reward:.3f})", obs.reward >= 0.5)) else: results.append(check("Perfect agent (no bugs)", True)) except Exception as e: results.append(check("perfect agent", False, str(e))) try: env = CodeReviewEnvironment() env.reset(seed=42) obs = env.step(_tool("submit_review", {})) results.append(check(f"Empty agent < 0.15 (got {obs.reward:.3f})", obs.reward < 0.15)) except Exception as e: results.append(check("empty agent", False, str(e))) # ── Packaging ── print("\n--- Packaging ---") try: with open("openenv.yaml") as f: config = yaml.safe_load(f) required = ["spec_version", "name", "type", "runtime", "app", "port"] results.append(check("openenv.yaml has required fields", all(k in config for k in required))) except Exception as e: results.append(check("openenv.yaml", False, str(e))) results.append(check("Dockerfile exists", os.path.exists("Dockerfile"))) # ── Summary ── print(f"\n{'=' * 60}") passed = sum(results) total = len(results) print(f"Results: {passed}/{total} checks passed") print("ALL CHECKS PASSED" if passed == total else f"{total - passed} checks FAILED") return 0 if passed == total else 1 if __name__ == "__main__": sys.exit(validate())