"""Engine tests for MissionCtrl environment. Covers: reset, action mechanics, hallucination injection, deterministic replay, score clamping, dependency graph, and edge cases. """ import sys import os sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from server.environment import ( MissionCtrlEngine, MissionCtrlEnvironment, parse_action, _clamp_score, DIFFICULTY_CONFIG, TASK_TIERS, AGENTS, HALLUCINATION_TYPES, ) def _approve_until_all_done(env: MissionCtrlEngine, max_steps: int = 80) -> None: """Approve tasks that have output, in dependency-safe order (each step unlocks children).""" steps = 0 while not env.done and steps < max_steps: steps += 1 ready = [t for t in env.tasks if t.status == "IN_PROGRESS" and t.output.strip()] if ready: env.step(f"APPROVE({ready[0].id})") else: env.step("NOOP") assert env.done, "expected episode to finish via APPROVE / NOOP" class TestScoreClamping: """All scores must be strictly in (0, 1) — never 0.0 or 1.0.""" def test_clamp_zero(self): assert _clamp_score(0.0) == 0.01 def test_clamp_one(self): assert _clamp_score(1.0) == 0.99 def test_clamp_negative(self): assert _clamp_score(-5.0) == 0.01 def test_clamp_above_one(self): assert _clamp_score(1.5) == 0.99 def test_clamp_normal(self): assert _clamp_score(0.5) == 0.5 def test_clamp_boundary_epsilon(self): result = _clamp_score(0.01) assert 0.0 < result < 1.0 def test_all_tasks_produce_valid_scores(self): """Every task tier must produce a score strictly in (0, 1).""" for task_id in ["easy", "medium", "hard", "special"]: env = MissionCtrlEngine() env.reset(task_id, seed=42) # Run a few NOOPs then grade for _ in range(3): env.step("NOOP") env.done = True score = env.grade() assert 0.0 < score < 1.0, f"Score for {task_id} out of range: {score}" class TestActionParser: """Action parser must handle all formats with NOOP fallback.""" def test_approve(self): p = parse_action("APPROVE(task_01)") assert p.action_type == "APPROVE" assert p.task_id == "task_01" def test_reject(self): p = parse_action('REJECT(task_02, "low quality output")') assert p.action_type == "REJECT" assert p.task_id == "task_02" assert "low quality" in p.reason def test_redelegate(self): p = parse_action("REDELEGATE(task_03, CoderAgent)") assert p.action_type == "REDELEGATE" assert p.task_id == "task_03" assert p.agent == "CoderAgent" def test_flag(self): p = parse_action('FLAG(task_04, "fabricated citation detected")') assert p.action_type == "FLAG" assert p.task_id == "task_04" assert "fabricated" in p.evidence def test_escalate(self): p = parse_action("ESCALATE(task_05)") assert p.action_type == "ESCALATE" assert p.task_id == "task_05" def test_synthesize(self): p = parse_action("SYNTHESIZE_REPORT()") assert p.action_type == "SYNTHESIZE_REPORT" def test_malformed_falls_back_to_noop(self): """CRITICAL: malformed input must NOT become SYNTHESIZE_REPORT.""" p = parse_action("garbage input blah blah") assert p.action_type == "NOOP" def test_empty_string_noop(self): p = parse_action("") assert p.action_type == "NOOP" def test_json_format(self): p = parse_action('{"action_type": "APPROVE", "task_id": "task_01"}') assert p.action_type == "APPROVE" assert p.task_id == "task_01" class TestEngineReset: """Reset must produce clean state.""" def test_reset_returns_observation(self): env = MissionCtrlEngine() obs = env.reset("easy", seed=42) assert "tasks" in obs assert "difficulty" in obs assert obs["done"] is False done_ids = {t["task_id"] for t in obs["tasks"] if t["status"] == "DONE"} for row in obs["tasks"]: tid = row["task_id"] deps = row.get("dependencies") or [] blocked = any(d not in done_ids for d in deps) if not blocked: assert row["status"] == "IN_PROGRESS", tid assert row["output"].strip(), tid else: assert row["status"] == "PENDING", tid assert row["output"] == "", tid def test_reset_tasks_match_tier(self): for tier in ["easy", "medium", "hard", "special"]: env = MissionCtrlEngine() obs = env.reset(tier, seed=42) num_tasks = DIFFICULTY_CONFIG[tier]["num_tasks"] assert len(obs["tasks"]) == num_tasks, f"{tier}: expected {num_tasks} tasks" def test_reset_clears_state(self): env = MissionCtrlEngine() env.reset("easy", seed=42) tid = env.tasks[0].id env.step(f"APPROVE({tid})") env.reset("medium", seed=99) assert env.time_step == 0 assert not env._flagged_ids assert not env._flag_results assert not env.done class TestHallucinationInjection: """Injection rates must match difficulty settings.""" def test_injections_occur(self): """At least some tasks should be injected across many seeds.""" total_injected = 0 env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) total_injected += len(env._injected_ids) # Hard = 65% rate with 4 tasks → expect ~260 injections over 100 episodes assert total_injected > 100, f"Too few injections: {total_injected}" def test_easy_has_fewer_injections(self): env_easy = MissionCtrlEngine() env_hard = MissionCtrlEngine() easy_count = 0 hard_count = 0 for s in range(50): env_easy.reset("easy", seed=s) easy_count += len(env_easy._ever_injected_ids) env_hard.reset("hard", seed=s) hard_count += len(env_hard._ever_injected_ids) assert easy_count < hard_count class TestActionMechanics: """Each action type must produce correct state transitions.""" def test_approve_sets_done(self): env = MissionCtrlEngine() env.reset("easy", seed=42) task_id = env.tasks[0].id env.step(f"APPROVE({task_id})") assert env.tasks[0].status == "DONE" def test_reject_then_regenerates_same_step(self): env = MissionCtrlEngine() env.reset("easy", seed=42) task_id = env.tasks[0].id env.step(f'REJECT({task_id}, "bad quality")') t = next(x for x in env.tasks if x.id == task_id) assert t.status == "IN_PROGRESS" assert t.output.strip() != "" assert t.hallucination_type is None or t.hallucination_type in HALLUCINATION_TYPES def test_flag_true_positive(self): """Flagging an injected task should be TP.""" env = MissionCtrlEngine() # Find a seed that produces an injection for seed in range(100): env.reset("hard", seed=seed) if env._injected_ids: injected_id = list(env._injected_ids)[0] _, reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")') assert reward > 0, "True positive should give positive reward" assert env._flag_results[injected_id] == "TP" return assert False, "No injection found in 100 seeds" def test_flag_false_positive(self): """Flagging a clean task should be FP.""" env = MissionCtrlEngine() env.reset("easy", seed=42) clean_ids = [t.id for t in env.tasks if t.hallucination_type is None] if clean_ids: _, reward, _, _ = env.step(f'FLAG({clean_ids[0]}, "suspected issue")') assert reward < 0, "False positive should give negative reward" def test_duplicate_flag_does_not_award_true_positive_again(self): """Repeated FLAG on the same task should not farm positive reward.""" env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._injected_ids: injected_id = list(env._injected_ids)[0] _, first_reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")') _, second_reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")') assert first_reward > 0, "Initial true positive should be rewarded" assert second_reward <= 0, "Duplicate FLAG should not be rewarded again" return assert False, "No injection found in 100 seeds" def test_escalate_blocks_task(self): env = MissionCtrlEngine() env.reset("easy", seed=42) task_id = env.tasks[0].id env.step(f"ESCALATE({task_id})") assert env.tasks[0].status == "BLOCKED" def test_synthesize_without_catching_all_penalized(self): """SYNTHESIZE_REPORT with uncaught hallucinations = penalty.""" env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._injected_ids: _, reward, _, _ = env.step("SYNTHESIZE_REPORT()") assert reward < 0, "Premature synthesis should be penalized" return def test_noop_small_penalty(self): env = MissionCtrlEngine() env.reset("easy", seed=42) _, reward, _, _ = env.step("NOOP") assert reward < 0 # small penalty for wasting step class TestDeterminism: """Same seed + same actions = same trajectory.""" def test_deterministic_replay(self): env1 = MissionCtrlEngine() env2 = MissionCtrlEngine() obs1 = env1.reset("medium", seed=123) obs2 = env2.reset("medium", seed=123) assert obs1 == obs2, "Same seed should produce identical observations" # Same sequence of actions actions = ["NOOP", "NOOP", "NOOP"] for action in actions: r1 = env1.step(action) r2 = env2.step(action) assert r1[1] == r2[1], "Same actions should produce same rewards" class TestEasyDifficultyPenaltyGate: """Passive FP penalty must be suppressed on easy difficulty.""" def test_easy_no_passive_penalty(self): env = MissionCtrlEngine() env.reset("easy", seed=42) # Don't flag anything — on easy, should NOT get passive penalty fp_signal = env._signal_false_positive_penalty() assert fp_signal == 0.0, f"Easy should suppress passive penalty, got {fp_signal}" def test_hard_gets_passive_penalty_when_no_flags(self): env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._ever_injected_ids: # Don't flag anything — should get passive penalty on hard fp_signal = env._signal_false_positive_penalty() assert fp_signal == 0.4, f"Hard should have passive penalty, got {fp_signal}" return class TestEpisodeBoundaries: """Episode termination conditions.""" def test_episode_ends_when_all_tasks_done(self): env = MissionCtrlEngine() env.reset("easy", seed=42) _approve_until_all_done(env) assert env.done def test_episode_ends_at_max_steps(self): env = MissionCtrlEngine() env.reset("easy", seed=42) for _ in range(20): env.step("NOOP") assert env.done def test_step_after_done_returns_zero_reward(self): env = MissionCtrlEngine() env.reset("easy", seed=42) env.done = True _, reward, done, _ = env.step("NOOP") assert reward == 0.0 assert done is True class TestRewardSignals: """Individual reward signal correctness.""" def test_task_completion_perfect(self): env = MissionCtrlEngine() env.reset("easy", seed=42) for t in env.tasks: if t.hallucination_type is not None: # Flag first, then approve env.step(f'FLAG({t.id}, "hallucination detected")') env.step(f"APPROVE({t.id})") score = env._signal_task_completion() assert score == 1.0 def test_task_completion_gives_partial_credit_for_true_positive_flag(self): env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._injected_ids: injected_id = sorted(env._injected_ids)[0] env.step(f'FLAG({injected_id}, "detected")') score = env._signal_task_completion() assert score == 0.5 / len(env.tasks) return assert False, "No injection found in 100 seeds" def test_task_completion_does_not_give_partial_credit_for_clean_unfinished_task(self): env = MissionCtrlEngine() env.reset("easy", seed=42) score = env._signal_task_completion() assert score == 0.0 def test_hallucination_detection_perfect(self): env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._ever_injected_ids: for iid in list(env._ever_injected_ids): env.step(f'FLAG({iid}, "detected")') dr = env._signal_hallucination_detection() assert dr == 1.0 return def test_no_hallucinations_returns_perfect_detection(self): """When no hallucinations exist, detection rate = 1.0.""" env = MissionCtrlEngine() env._injected_ids = set() env._ever_injected_ids = set() assert env._signal_hallucination_detection() == 1.0 class TestDependencyUnlock: """Dependency-gated outputs and unlock ordering.""" def test_roots_in_progress_after_reset(self): env = MissionCtrlEngine() env.reset("easy", seed=42) for t in env.tasks: if not t.dependencies: assert t.status == "IN_PROGRESS" assert t.output.strip() def test_pruned_dependency_outside_sample_unlocks_at_reset(self): """Deps pointing outside the sampled set are dropped — task is a root.""" for seed in range(300): env = MissionCtrlEngine() env.reset("medium", seed=seed) ids = {t.id for t in env.tasks} if "task_07" in ids and "task_06" not in ids: t7 = next(t for t in env.tasks if t.id == "task_07") assert t7.dependencies == [] assert t7.status == "IN_PROGRESS" assert t7.output.strip() return raise AssertionError("no seed with task_07 without task_06 in 300 tries") def test_chain_unlocks_only_after_parents_done(self): for seed in range(500): env = MissionCtrlEngine() env.reset("medium", seed=seed) ids = {t.id for t in env.tasks} if {"task_06", "task_07", "task_08"}.issubset(ids): t6 = next(t for t in env.tasks if t.id == "task_06") t7 = next(t for t in env.tasks if t.id == "task_07") t8 = next(t for t in env.tasks if t.id == "task_08") assert t6.status == "IN_PROGRESS" and t6.output assert t7.status == "PENDING" and t7.output == "" assert t8.status == "PENDING" and t8.output == "" env.step("APPROVE(task_06)") t7 = next(t for t in env.tasks if t.id == "task_07") t8 = next(t for t in env.tasks if t.id == "task_08") assert t7.status == "IN_PROGRESS" and t7.output assert t8.status == "PENDING" and t8.output == "" env.step("APPROVE(task_07)") t8 = next(t for t in env.tasks if t.id == "task_08") assert t8.status == "IN_PROGRESS" and t8.output.strip() return raise AssertionError("no medium seed with task_06,07,08 in 500 tries") class TestRegenerationHallucination: """Reject / redelegate clears corruption metadata and regen path.""" def test_approve_after_redelegate_clean_has_no_hall_penalty(self): for seed in range(200): env = MissionCtrlEngine() env.reset("hard", seed=seed) injected = [t for t in env.tasks if t.hallucination_type is not None] if not injected: continue t = injected[0] alt = next(a for a in AGENTS if a != t.assigned_agent) env.step(f"REDELEGATE({t.id}, {alt})") t2 = next(x for x in env.tasks if x.id == t.id) if t2.hallucination_type is not None: continue _, r, _, _ = env.step(f"APPROVE({t.id})") assert r == 1.0 return raise AssertionError("no suitable seed for clean redelegate") def test_redelegate_invalid_agent_no_mutation(self): env = MissionCtrlEngine() env.reset("easy", seed=42) t0 = env.tasks[0] before = (t0.status, t0.output, t0.assigned_agent) _, r, _, _ = env.step(f"REDELEGATE({t0.id}, NonexistentAgent)") assert r == -0.5 assert (t0.status, t0.output, t0.assigned_agent) == before class TestSynthesizeEdgeCases: """SYNTHESIZE_REPORT gating on current corruption.""" def test_synthesize_allowed_when_no_active_hallucination(self): for seed in range(80): env = MissionCtrlEngine() env.reset("easy", seed=seed) if env._injected_ids: continue _, reward, done, _ = env.step("SYNTHESIZE_REPORT()") assert reward > 0 assert done return raise AssertionError("no easy seed without injection in 80 tries") def test_synthesize_penalized_when_hall_uncaught(self): env = MissionCtrlEngine() for seed in range(100): env.reset("hard", seed=seed) if env._injected_ids: _, reward, _, _ = env.step("SYNTHESIZE_REPORT()") assert reward < 0 return raise AssertionError("No injection found in 100 seeds") class TestInvalidActions: """Invalid task ids and illegal approve targets.""" def test_approve_unknown_task_negative_reward(self): env = MissionCtrlEngine() env.reset("easy", seed=42) _, r, _, _ = env.step("APPROVE(task_nonexistent)") assert r == -1.0 def test_approve_pending_child_negative_reward(self): for seed in range(500): env = MissionCtrlEngine() env.reset("medium", seed=seed) ids = {t.id for t in env.tasks} if {"task_06", "task_07"}.issubset(ids): _, r, _, _ = env.step("APPROVE(task_07)") assert r == -0.5 return raise AssertionError("no suitable medium seed") class TestMissionCtrlEnvironmentWrapper: """HTTP wrapper contract.""" def test_reset_step_payload_keys(self): env = MissionCtrlEnvironment() out = env.reset("easy", seed=1) assert set(out.keys()) == {"observation", "done"} assert "tasks" in out["observation"] step_out = env.step("NOOP") assert set(step_out.keys()) == {"observation", "reward", "done", "info"}