missionctrl_env / tests /test_engine.py
Jit-fnc's picture
Upload folder using huggingface_hub
95af320 verified
Raw History Blame Contribute Delete
19.7 kB
"""Engine tests for MissionCtrl environment.
Covers: reset, action mechanics, hallucination injection, deterministic replay,
score clamping, dependency graph, and edge cases.
"""
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from server.environment import (
MissionCtrlEngine, MissionCtrlEnvironment,
parse_action, _clamp_score, DIFFICULTY_CONFIG, TASK_TIERS,
AGENTS, HALLUCINATION_TYPES,
)
def _approve_until_all_done(env: MissionCtrlEngine, max_steps: int = 80) -> None:
"""Approve tasks that have output, in dependency-safe order (each step unlocks children)."""
steps = 0
while not env.done and steps < max_steps:
steps += 1
ready = [t for t in env.tasks if t.status == "IN_PROGRESS" and t.output.strip()]
if ready:
env.step(f"APPROVE({ready[0].id})")
else:
env.step("NOOP")
assert env.done, "expected episode to finish via APPROVE / NOOP"
class TestScoreClamping:
"""All scores must be strictly in (0, 1) — never 0.0 or 1.0."""
def test_clamp_zero(self):
assert _clamp_score(0.0) == 0.01
def test_clamp_one(self):
assert _clamp_score(1.0) == 0.99
def test_clamp_negative(self):
assert _clamp_score(-5.0) == 0.01
def test_clamp_above_one(self):
assert _clamp_score(1.5) == 0.99
def test_clamp_normal(self):
assert _clamp_score(0.5) == 0.5
def test_clamp_boundary_epsilon(self):
result = _clamp_score(0.01)
assert 0.0 < result < 1.0
def test_all_tasks_produce_valid_scores(self):
"""Every task tier must produce a score strictly in (0, 1)."""
for task_id in ["easy", "medium", "hard", "special"]:
env = MissionCtrlEngine()
env.reset(task_id, seed=42)
# Run a few NOOPs then grade
for _ in range(3):
env.step("NOOP")
env.done = True
score = env.grade()
assert 0.0 < score < 1.0, f"Score for {task_id} out of range: {score}"
class TestActionParser:
"""Action parser must handle all formats with NOOP fallback."""
def test_approve(self):
p = parse_action("APPROVE(task_01)")
assert p.action_type == "APPROVE"
assert p.task_id == "task_01"
def test_reject(self):
p = parse_action('REJECT(task_02, "low quality output")')
assert p.action_type == "REJECT"
assert p.task_id == "task_02"
assert "low quality" in p.reason
def test_redelegate(self):
p = parse_action("REDELEGATE(task_03, CoderAgent)")
assert p.action_type == "REDELEGATE"
assert p.task_id == "task_03"
assert p.agent == "CoderAgent"
def test_flag(self):
p = parse_action('FLAG(task_04, "fabricated citation detected")')
assert p.action_type == "FLAG"
assert p.task_id == "task_04"
assert "fabricated" in p.evidence
def test_escalate(self):
p = parse_action("ESCALATE(task_05)")
assert p.action_type == "ESCALATE"
assert p.task_id == "task_05"
def test_synthesize(self):
p = parse_action("SYNTHESIZE_REPORT()")
assert p.action_type == "SYNTHESIZE_REPORT"
def test_malformed_falls_back_to_noop(self):
"""CRITICAL: malformed input must NOT become SYNTHESIZE_REPORT."""
p = parse_action("garbage input blah blah")
assert p.action_type == "NOOP"
def test_empty_string_noop(self):
p = parse_action("")
assert p.action_type == "NOOP"
def test_json_format(self):
p = parse_action('{"action_type": "APPROVE", "task_id": "task_01"}')
assert p.action_type == "APPROVE"
assert p.task_id == "task_01"
class TestEngineReset:
"""Reset must produce clean state."""
def test_reset_returns_observation(self):
env = MissionCtrlEngine()
obs = env.reset("easy", seed=42)
assert "tasks" in obs
assert "difficulty" in obs
assert obs["done"] is False
done_ids = {t["task_id"] for t in obs["tasks"] if t["status"] == "DONE"}
for row in obs["tasks"]:
tid = row["task_id"]
deps = row.get("dependencies") or []
blocked = any(d not in done_ids for d in deps)
if not blocked:
assert row["status"] == "IN_PROGRESS", tid
assert row["output"].strip(), tid
else:
assert row["status"] == "PENDING", tid
assert row["output"] == "", tid
def test_reset_tasks_match_tier(self):
for tier in ["easy", "medium", "hard", "special"]:
env = MissionCtrlEngine()
obs = env.reset(tier, seed=42)
num_tasks = DIFFICULTY_CONFIG[tier]["num_tasks"]
assert len(obs["tasks"]) == num_tasks, f"{tier}: expected {num_tasks} tasks"
def test_reset_clears_state(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
tid = env.tasks[0].id
env.step(f"APPROVE({tid})")
env.reset("medium", seed=99)
assert env.time_step == 0
assert not env._flagged_ids
assert not env._flag_results
assert not env.done
class TestHallucinationInjection:
"""Injection rates must match difficulty settings."""
def test_injections_occur(self):
"""At least some tasks should be injected across many seeds."""
total_injected = 0
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
total_injected += len(env._injected_ids)
# Hard = 65% rate with 4 tasks → expect ~260 injections over 100 episodes
assert total_injected > 100, f"Too few injections: {total_injected}"
def test_easy_has_fewer_injections(self):
env_easy = MissionCtrlEngine()
env_hard = MissionCtrlEngine()
easy_count = 0
hard_count = 0
for s in range(50):
env_easy.reset("easy", seed=s)
easy_count += len(env_easy._ever_injected_ids)
env_hard.reset("hard", seed=s)
hard_count += len(env_hard._ever_injected_ids)
assert easy_count < hard_count
class TestActionMechanics:
"""Each action type must produce correct state transitions."""
def test_approve_sets_done(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
task_id = env.tasks[0].id
env.step(f"APPROVE({task_id})")
assert env.tasks[0].status == "DONE"
def test_reject_then_regenerates_same_step(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
task_id = env.tasks[0].id
env.step(f'REJECT({task_id}, "bad quality")')
t = next(x for x in env.tasks if x.id == task_id)
assert t.status == "IN_PROGRESS"
assert t.output.strip() != ""
assert t.hallucination_type is None or t.hallucination_type in HALLUCINATION_TYPES
def test_flag_true_positive(self):
"""Flagging an injected task should be TP."""
env = MissionCtrlEngine()
# Find a seed that produces an injection
for seed in range(100):
env.reset("hard", seed=seed)
if env._injected_ids:
injected_id = list(env._injected_ids)[0]
_, reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")')
assert reward > 0, "True positive should give positive reward"
assert env._flag_results[injected_id] == "TP"
return
assert False, "No injection found in 100 seeds"
def test_flag_false_positive(self):
"""Flagging a clean task should be FP."""
env = MissionCtrlEngine()
env.reset("easy", seed=42)
clean_ids = [t.id for t in env.tasks if t.hallucination_type is None]
if clean_ids:
_, reward, _, _ = env.step(f'FLAG({clean_ids[0]}, "suspected issue")')
assert reward < 0, "False positive should give negative reward"
def test_duplicate_flag_does_not_award_true_positive_again(self):
"""Repeated FLAG on the same task should not farm positive reward."""
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._injected_ids:
injected_id = list(env._injected_ids)[0]
_, first_reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")')
_, second_reward, _, _ = env.step(f'FLAG({injected_id}, "detected hallucination")')
assert first_reward > 0, "Initial true positive should be rewarded"
assert second_reward <= 0, "Duplicate FLAG should not be rewarded again"
return
assert False, "No injection found in 100 seeds"
def test_escalate_blocks_task(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
task_id = env.tasks[0].id
env.step(f"ESCALATE({task_id})")
assert env.tasks[0].status == "BLOCKED"
def test_synthesize_without_catching_all_penalized(self):
"""SYNTHESIZE_REPORT with uncaught hallucinations = penalty."""
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._injected_ids:
_, reward, _, _ = env.step("SYNTHESIZE_REPORT()")
assert reward < 0, "Premature synthesis should be penalized"
return
def test_noop_small_penalty(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
_, reward, _, _ = env.step("NOOP")
assert reward < 0 # small penalty for wasting step
class TestDeterminism:
"""Same seed + same actions = same trajectory."""
def test_deterministic_replay(self):
env1 = MissionCtrlEngine()
env2 = MissionCtrlEngine()
obs1 = env1.reset("medium", seed=123)
obs2 = env2.reset("medium", seed=123)
assert obs1 == obs2, "Same seed should produce identical observations"
# Same sequence of actions
actions = ["NOOP", "NOOP", "NOOP"]
for action in actions:
r1 = env1.step(action)
r2 = env2.step(action)
assert r1[1] == r2[1], "Same actions should produce same rewards"
class TestEasyDifficultyPenaltyGate:
"""Passive FP penalty must be suppressed on easy difficulty."""
def test_easy_no_passive_penalty(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
# Don't flag anything — on easy, should NOT get passive penalty
fp_signal = env._signal_false_positive_penalty()
assert fp_signal == 0.0, f"Easy should suppress passive penalty, got {fp_signal}"
def test_hard_gets_passive_penalty_when_no_flags(self):
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._ever_injected_ids:
# Don't flag anything — should get passive penalty on hard
fp_signal = env._signal_false_positive_penalty()
assert fp_signal == 0.4, f"Hard should have passive penalty, got {fp_signal}"
return
class TestEpisodeBoundaries:
"""Episode termination conditions."""
def test_episode_ends_when_all_tasks_done(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
_approve_until_all_done(env)
assert env.done
def test_episode_ends_at_max_steps(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
for _ in range(20):
env.step("NOOP")
assert env.done
def test_step_after_done_returns_zero_reward(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
env.done = True
_, reward, done, _ = env.step("NOOP")
assert reward == 0.0
assert done is True
class TestRewardSignals:
"""Individual reward signal correctness."""
def test_task_completion_perfect(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
for t in env.tasks:
if t.hallucination_type is not None:
# Flag first, then approve
env.step(f'FLAG({t.id}, "hallucination detected")')
env.step(f"APPROVE({t.id})")
score = env._signal_task_completion()
assert score == 1.0
def test_task_completion_gives_partial_credit_for_true_positive_flag(self):
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._injected_ids:
injected_id = sorted(env._injected_ids)[0]
env.step(f'FLAG({injected_id}, "detected")')
score = env._signal_task_completion()
assert score == 0.5 / len(env.tasks)
return
assert False, "No injection found in 100 seeds"
def test_task_completion_does_not_give_partial_credit_for_clean_unfinished_task(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
score = env._signal_task_completion()
assert score == 0.0
def test_hallucination_detection_perfect(self):
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._ever_injected_ids:
for iid in list(env._ever_injected_ids):
env.step(f'FLAG({iid}, "detected")')
dr = env._signal_hallucination_detection()
assert dr == 1.0
return
def test_no_hallucinations_returns_perfect_detection(self):
"""When no hallucinations exist, detection rate = 1.0."""
env = MissionCtrlEngine()
env._injected_ids = set()
env._ever_injected_ids = set()
assert env._signal_hallucination_detection() == 1.0
class TestDependencyUnlock:
"""Dependency-gated outputs and unlock ordering."""
def test_roots_in_progress_after_reset(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
for t in env.tasks:
if not t.dependencies:
assert t.status == "IN_PROGRESS"
assert t.output.strip()
def test_pruned_dependency_outside_sample_unlocks_at_reset(self):
"""Deps pointing outside the sampled set are dropped — task is a root."""
for seed in range(300):
env = MissionCtrlEngine()
env.reset("medium", seed=seed)
ids = {t.id for t in env.tasks}
if "task_07" in ids and "task_06" not in ids:
t7 = next(t for t in env.tasks if t.id == "task_07")
assert t7.dependencies == []
assert t7.status == "IN_PROGRESS"
assert t7.output.strip()
return
raise AssertionError("no seed with task_07 without task_06 in 300 tries")
def test_chain_unlocks_only_after_parents_done(self):
for seed in range(500):
env = MissionCtrlEngine()
env.reset("medium", seed=seed)
ids = {t.id for t in env.tasks}
if {"task_06", "task_07", "task_08"}.issubset(ids):
t6 = next(t for t in env.tasks if t.id == "task_06")
t7 = next(t for t in env.tasks if t.id == "task_07")
t8 = next(t for t in env.tasks if t.id == "task_08")
assert t6.status == "IN_PROGRESS" and t6.output
assert t7.status == "PENDING" and t7.output == ""
assert t8.status == "PENDING" and t8.output == ""
env.step("APPROVE(task_06)")
t7 = next(t for t in env.tasks if t.id == "task_07")
t8 = next(t for t in env.tasks if t.id == "task_08")
assert t7.status == "IN_PROGRESS" and t7.output
assert t8.status == "PENDING" and t8.output == ""
env.step("APPROVE(task_07)")
t8 = next(t for t in env.tasks if t.id == "task_08")
assert t8.status == "IN_PROGRESS" and t8.output.strip()
return
raise AssertionError("no medium seed with task_06,07,08 in 500 tries")
class TestRegenerationHallucination:
"""Reject / redelegate clears corruption metadata and regen path."""
def test_approve_after_redelegate_clean_has_no_hall_penalty(self):
for seed in range(200):
env = MissionCtrlEngine()
env.reset("hard", seed=seed)
injected = [t for t in env.tasks if t.hallucination_type is not None]
if not injected:
continue
t = injected[0]
alt = next(a for a in AGENTS if a != t.assigned_agent)
env.step(f"REDELEGATE({t.id}, {alt})")
t2 = next(x for x in env.tasks if x.id == t.id)
if t2.hallucination_type is not None:
continue
_, r, _, _ = env.step(f"APPROVE({t.id})")
assert r == 1.0
return
raise AssertionError("no suitable seed for clean redelegate")
def test_redelegate_invalid_agent_no_mutation(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
t0 = env.tasks[0]
before = (t0.status, t0.output, t0.assigned_agent)
_, r, _, _ = env.step(f"REDELEGATE({t0.id}, NonexistentAgent)")
assert r == -0.5
assert (t0.status, t0.output, t0.assigned_agent) == before
class TestSynthesizeEdgeCases:
"""SYNTHESIZE_REPORT gating on current corruption."""
def test_synthesize_allowed_when_no_active_hallucination(self):
for seed in range(80):
env = MissionCtrlEngine()
env.reset("easy", seed=seed)
if env._injected_ids:
continue
_, reward, done, _ = env.step("SYNTHESIZE_REPORT()")
assert reward > 0
assert done
return
raise AssertionError("no easy seed without injection in 80 tries")
def test_synthesize_penalized_when_hall_uncaught(self):
env = MissionCtrlEngine()
for seed in range(100):
env.reset("hard", seed=seed)
if env._injected_ids:
_, reward, _, _ = env.step("SYNTHESIZE_REPORT()")
assert reward < 0
return
raise AssertionError("No injection found in 100 seeds")
class TestInvalidActions:
"""Invalid task ids and illegal approve targets."""
def test_approve_unknown_task_negative_reward(self):
env = MissionCtrlEngine()
env.reset("easy", seed=42)
_, r, _, _ = env.step("APPROVE(task_nonexistent)")
assert r == -1.0
def test_approve_pending_child_negative_reward(self):
for seed in range(500):
env = MissionCtrlEngine()
env.reset("medium", seed=seed)
ids = {t.id for t in env.tasks}
if {"task_06", "task_07"}.issubset(ids):
_, r, _, _ = env.step("APPROVE(task_07)")
assert r == -0.5
return
raise AssertionError("no suitable medium seed")
class TestMissionCtrlEnvironmentWrapper:
"""HTTP wrapper contract."""
def test_reset_step_payload_keys(self):
env = MissionCtrlEnvironment()
out = env.reset("easy", seed=1)
assert set(out.keys()) == {"observation", "done"}
assert "tasks" in out["observation"]
step_out = env.step("NOOP")
assert set(step_out.keys()) == {"observation", "reward", "done", "info"}