Download tests/test_phase1_e2e.py from polyHedra/PolyHedra: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/spaces/polyHedra/PolyHedra/resolve/main/tests/test_phase1_e2e.py
- Command line
-
hf download hf://spaces/polyHedra/PolyHedra/tests/test_phase1_e2e.py
-
curl -L -o test_phase1_e2e.py https://huggingface.co/spaces/polyHedra/PolyHedra/resolve/main/tests/test_phase1_e2e.py
21.5 kB
| """Phase 1 exit-criteria integration test. | |
| Verifies the full flow mandated by the PRD Phase 1 exit gate: | |
| A user can start a session, have Scout search for papers, and have | |
| Strategist propose research directions with a checkpoint gate. | |
| All external I/O is mocked so the test runs offline and deterministically. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| from pathlib import Path | |
| from tempfile import TemporaryDirectory | |
| from typing import Any | |
| from unittest.mock import AsyncMock, MagicMock, patch | |
| import pytest | |
| from polyhedra.agents.base import Mention | |
| from polyhedra.agents.coordinator import CheckpointResult, CoordinatorService | |
| from polyhedra.agents.critic import CriticAgent | |
| from polyhedra.agents.scout import ScoutAgent | |
| from polyhedra.agents.scribe import ScribeAgent | |
| from polyhedra.agents.strategist import StrategistAgent | |
| from polyhedra.core.role_registry import RoleEntry | |
| from polyhedra.core.session import SessionManager | |
| # --------------------------------------------------------------------------- # | |
| # Shared test fixtures | |
| # --------------------------------------------------------------------------- # | |
| FAKE_PAPERS = [ | |
| { | |
| "paperId": f"paper{i:03d}", | |
| "title": f"Efficient Transformers: Approach {i}", | |
| "authors": [{"name": f"Author {i}"}], | |
| "year": 2022 + (i % 3), | |
| "venue": "NeurIPS", | |
| "abstract": f"We propose approach {i} for efficient attention in long contexts.", | |
| "citationCount": 100 + i * 10, | |
| "relevanceScore": 0.9 - i * 0.02, | |
| } | |
| for i in range(1, 21) | |
| ] | |
| FAKE_LLM_STRATEGIST = """\ | |
| ### Direction 1: Sparse-Linear Hybrid Attention | |
| **Gap:** No work combines sparse and linear attention in a stable training setup. | |
| **Hypothesis:** Sparse-linear hybrid reduces compute by 40% on sequences >8k tokens. | |
| **Methodology sketch:** Implement token routing before attention head; train with gradient norm clipping. | |
| **Novelty:** high | |
| **Feasibility:** medium | |
| **Confidence:** 0.82 | |
| ### Direction 2: Adaptive Context Windows | |
| **Gap:** Fixed context windows waste compute on short-range dependencies. | |
| **Hypothesis:** Adaptive windows achieve parity with full attention at 60% cost. | |
| **Methodology sketch:** Learn per-head dynamic span; benchmark on LongBench. | |
| **Novelty:** medium | |
| **Feasibility:** high | |
| **Confidence:** 0.75 | |
| ### Direction 3: Retrieval-Augmented Long Context | |
| **Gap:** Current RAG ignores intra-document structure. | |
| **Hypothesis:** Structure-aware RAG improves perplexity by 5% on BookSum. | |
| **Methodology sketch:** Graph-based document chunking with cross-chunk attention. | |
| **Novelty:** low | |
| **Feasibility:** high | |
| **Confidence:** 0.68 | |
| """ | |
| FAKE_LLM_CRITIC = """\ | |
| **Strengths** | |
| - Direction 1 targets a genuine gap with a clear metric (40% compute reduction). | |
| - Methodology sketch is actionable. | |
| **Weaknesses** | |
| - Confidence 0.82 is optimistic; sparse-linear training is historically unstable. | |
| - No baseline comparison specified. | |
| **Missing** | |
| - Citations for sparse attention instability literature. | |
| - Ablation plan for the routing mechanism. | |
| **Path to publishable** | |
| - Add a training stability section with gradient norm analysis. | |
| - Compare against FlashAttention-2 baseline explicitly. | |
| """ | |
| def tmp_sessions(tmp_path: Path) -> SessionManager: | |
| return SessionManager(tmp_path / "sessions") | |
| def mock_scholar() -> MagicMock: | |
| """Mock SemanticScholarService.search to return FAKE_PAPERS.""" | |
| m = AsyncMock(return_value=FAKE_PAPERS) | |
| return m | |
| def mock_llm_factory(): | |
| """Return a factory that produces an LLMService mock with a given response.""" | |
| def _factory(response: str) -> MagicMock: | |
| m = MagicMock() | |
| m.complete = AsyncMock(return_value=(response, 500, 200)) | |
| return m | |
| return _factory | |
| # --------------------------------------------------------------------------- # | |
| # Helpers | |
| # --------------------------------------------------------------------------- # | |
| def _build_service( | |
| tmp_sessions: SessionManager, | |
| goal: str, | |
| scout: ScoutAgent, | |
| strategist: StrategistAgent, | |
| critic: CriticAgent, | |
| scribe: ScribeAgent, | |
| ) -> tuple[Any, list[tuple[str, str]], list[str]]: | |
| """Create a session + CoordinatorService wired to capture posts and checkpoints.""" | |
| session = tmp_sessions.create(goal=goal, platform="cli") | |
| for role_name, agent in [ | |
| ("scout", scout), | |
| ("strategist", strategist), | |
| ("critic", critic), | |
| ("scribe", scribe), | |
| ]: | |
| session.registry.register(RoleEntry(role=role_name, agent=agent)) | |
| posts: list[tuple[str, str]] = [] | |
| checkpoints: list[str] = [] | |
| async def post_fn(from_role: str, content: str) -> None: | |
| posts.append((from_role, content)) | |
| async def checkpoint_fn(summary: str) -> CheckpointResult: | |
| checkpoints.append(summary) | |
| return CheckpointResult(approved=True, response="yes") | |
| service = CoordinatorService( | |
| session=session, | |
| post_fn=post_fn, | |
| checkpoint_fn=checkpoint_fn, | |
| ) | |
| return service, posts, checkpoints | |
| # --------------------------------------------------------------------------- # | |
| # Tests | |
| # --------------------------------------------------------------------------- # | |
| class TestPhase1ExitCriteria: | |
| """Phase 1 exit gate: init β Scout β checkpoint β Strategist β checkpoint.""" | |
| async def test_session_initialization(self, tmp_sessions: SessionManager) -> None: | |
| """Coordinator init posts a welcome message with the goal.""" | |
| scout = ScoutAgent() | |
| session = tmp_sessions.create(goal="Efficient transformers", platform="cli") | |
| session.registry.register(RoleEntry(role="scout", agent=scout)) | |
| posts: list[tuple[str, str]] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| async def chk_fn(summary: str) -> CheckpointResult: | |
| return CheckpointResult(approved=True) | |
| service = CoordinatorService(session=session, post_fn=post_fn, checkpoint_fn=chk_fn) | |
| await service.initialize_session( | |
| goal="Efficient transformers", | |
| selected_roles=["scout"], | |
| human_roles=[], | |
| ) | |
| assert any("coordinator" == role for role, _ in posts), "Coordinator must post welcome" | |
| welcome_text = next(c for r, c in posts if r == "coordinator") | |
| assert "Efficient transformers" in welcome_text | |
| assert session.goal == "Efficient transformers" | |
| async def test_scout_search_flow( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| mock_llm_factory: Any, | |
| ) -> None: | |
| """@scout mention β papers found β checkpoint fires β @strategist mentioned.""" | |
| strategist_llm = mock_llm_factory(FAKE_LLM_STRATEGIST) | |
| scout = ScoutAgent() | |
| strategist = StrategistAgent(llm_service=strategist_llm) | |
| critic = CriticAgent(llm_service=mock_llm_factory(FAKE_LLM_CRITIC)) | |
| scribe = ScribeAgent() | |
| service, posts, checkpoints = _build_service( | |
| tmp_sessions, | |
| goal="Efficient transformers for long-context NLP", | |
| scout=scout, | |
| strategist=strategist, | |
| critic=critic, | |
| scribe=scribe, | |
| ) | |
| # Patch SemanticScholarService.search on the scout's instance | |
| scout._scholar.search = mock_scholar | |
| # Human sends @scout mention | |
| await service.dispatch( | |
| from_role="human", | |
| text="@scout search for efficient transformers for long-context NLP", | |
| ) | |
| # Scout must have posted | |
| assert any(r == "scout" for r, _ in posts), "Scout must post a response" | |
| scout_post = next(c for r, c in posts if r == "scout") | |
| assert "Scout" in scout_post | |
| assert len(FAKE_PAPERS) == 20 # sanity - our fixture has 20 papers | |
| # Checkpoint must have fired (Scout triggers one after returning results) | |
| assert len(checkpoints) >= 1, "At least one checkpoint must fire" | |
| async def test_full_phase1_chain( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| mock_llm_factory: Any, | |
| ) -> None: | |
| """ | |
| Full Phase 1 exit-criteria chain: | |
| @scout β papers β checkpoint(approved) β @strategist β directions β checkpoint. | |
| Because the Coordinator auto-dispatches @strategist after Scout's response, | |
| we expect both Scout and Strategist posts in a single dispatch call. | |
| """ | |
| strategist_llm = mock_llm_factory(FAKE_LLM_STRATEGIST) | |
| critic_llm = mock_llm_factory(FAKE_LLM_CRITIC) | |
| scout = ScoutAgent() | |
| strategist = StrategistAgent(llm_service=strategist_llm) | |
| critic = CriticAgent(llm_service=critic_llm) | |
| scribe = ScribeAgent() | |
| service, posts, checkpoints = _build_service( | |
| tmp_sessions, | |
| goal="Efficient transformers for long-context NLP", | |
| scout=scout, | |
| strategist=strategist, | |
| critic=critic, | |
| scribe=scribe, | |
| ) | |
| scout._scholar.search = mock_scholar | |
| await service.dispatch( | |
| from_role="human", | |
| text="@scout search for efficient transformers for long-context NLP", | |
| ) | |
| roles_that_posted = [r for r, _ in posts] | |
| # Scout must have posted | |
| assert "scout" in roles_that_posted, f"Scout did not post. Got: {roles_that_posted}" | |
| # Strategist must have posted (chained from Scout) | |
| assert "strategist" in roles_that_posted, f"Strategist did not post. Got: {roles_that_posted}" | |
| # At least 2 checkpoints: one after Scout, one after Strategist | |
| assert len(checkpoints) >= 2, f"Expected β₯2 checkpoints, got {len(checkpoints)}: {checkpoints}" | |
| # Strategist content should contain research directions | |
| strategist_post = next(c for r, c in posts if r == "strategist") | |
| assert "Direction" in strategist_post | |
| async def test_checkpoint_stop_halts_chain( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| mock_llm_factory: Any, | |
| ) -> None: | |
| """If user stops at Scout checkpoint, Strategist must NOT run.""" | |
| scout = ScoutAgent() | |
| strategist = StrategistAgent(llm_service=mock_llm_factory(FAKE_LLM_STRATEGIST)) | |
| critic = CriticAgent(llm_service=mock_llm_factory(FAKE_LLM_CRITIC)) | |
| scribe = ScribeAgent() | |
| session = tmp_sessions.create(goal="Test halt", platform="cli") | |
| for role_name, agent in [ | |
| ("scout", scout), ("strategist", strategist), | |
| ("critic", critic), ("scribe", scribe), | |
| ]: | |
| session.registry.register(RoleEntry(role=role_name, agent=agent)) | |
| posts: list[tuple[str, str]] = [] | |
| checkpoints: list[str] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| async def stop_at_first_checkpoint(summary: str) -> CheckpointResult: | |
| checkpoints.append(summary) | |
| # Reject the first checkpoint (user says "stop") | |
| return CheckpointResult(approved=False, response="stop") | |
| service = CoordinatorService( | |
| session=session, post_fn=post_fn, checkpoint_fn=stop_at_first_checkpoint | |
| ) | |
| scout._scholar.search = mock_scholar | |
| await service.dispatch( | |
| from_role="human", | |
| text="@scout search for sparse attention", | |
| ) | |
| roles_that_posted = [r for r, _ in posts] | |
| assert "scout" in roles_that_posted, "Scout must still post its results" | |
| assert "strategist" not in roles_that_posted, "Strategist must NOT run when checkpoint is rejected" | |
| # Coordinator should have posted a "paused" message | |
| coord_posts = [c for r, c in posts if r == "coordinator"] | |
| assert any("paused" in c.lower() or "stop" in c.lower() or "βΈ" in c for c in coord_posts), \ | |
| "Coordinator must post a pause/stop message" | |
| async def test_mention_loop_guard( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| ) -> None: | |
| """@mention chain exceeding depth 3 must be cut by Coordinator.""" | |
| # A pathological agent that always @mentions itself back | |
| class LoopAgent(ScoutAgent): | |
| async def handle(self, mention, thread_context, research_state): | |
| from polyhedra.agents.base import AgentResponse | |
| return AgentResponse( | |
| agent_role=self.role, | |
| content="@scout looping", | |
| mentions=["scout"], # always loops back to itself | |
| ) | |
| loop_agent = LoopAgent() | |
| session = tmp_sessions.create(goal="Loop test") | |
| session.registry.register(RoleEntry(role="scout", agent=loop_agent)) | |
| posts: list[tuple[str, str]] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| async def chk_fn(summary: str) -> CheckpointResult: | |
| return CheckpointResult(approved=True) | |
| service = CoordinatorService(session=session, post_fn=post_fn, checkpoint_fn=chk_fn) | |
| await service.dispatch(from_role="human", text="@scout go") | |
| # Loop must be broken β coordinator must have posted the limit message | |
| coord_posts = [c for r, c in posts if r == "coordinator"] | |
| assert any("limit" in c.lower() or "β" in c for c in coord_posts), \ | |
| f"Loop guard must fire. Coordinator posts: {coord_posts}" | |
| async def test_human_role_takeover( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| ) -> None: | |
| """When human holds Critic role, @critic mention is forwarded to human (not LLM).""" | |
| scout = ScoutAgent() | |
| critic = CriticAgent() | |
| session = tmp_sessions.create(goal="Role takeover test") | |
| session.registry.register(RoleEntry(role="scout", agent=scout)) | |
| session.registry.register(RoleEntry(role="critic", agent=critic)) | |
| # Human takes over Critic | |
| session.registry.take_over("critic") | |
| posts: list[tuple[str, str]] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| async def chk_fn(summary: str) -> CheckpointResult: | |
| return CheckpointResult(approved=True) | |
| service = CoordinatorService(session=session, post_fn=post_fn, checkpoint_fn=chk_fn) | |
| await service.dispatch(from_role="human", text="@critic please review this draft") | |
| # Coordinator must forward to human, not run agent | |
| coord_posts = [c for r, c in posts if r == "coordinator"] | |
| assert any("human" in c.lower() or "critic" in c.lower() for c in coord_posts), \ | |
| "Coordinator must forward @critic to the human role-holder" | |
| # Critic LLM was never called (role held by human) | |
| critic_posts = [c for r, c in posts if r == "critic"] | |
| assert len(critic_posts) == 0, "Critic agent must not post when role is held by human" | |
| async def test_research_state_updated_after_scout( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| mock_llm_factory: Any, | |
| ) -> None: | |
| """Research State must contain key papers after Scout returns results.""" | |
| scout = ScoutAgent() | |
| strategist = StrategistAgent(llm_service=mock_llm_factory(FAKE_LLM_STRATEGIST)) | |
| critic = CriticAgent(llm_service=mock_llm_factory(FAKE_LLM_CRITIC)) | |
| scribe = ScribeAgent() | |
| session = tmp_sessions.create(goal="State update test") | |
| for role_name, agent in [ | |
| ("scout", scout), ("strategist", strategist), | |
| ("critic", critic), ("scribe", scribe), | |
| ]: | |
| session.registry.register(RoleEntry(role=role_name, agent=agent)) | |
| posts: list[tuple[str, str]] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| async def chk_fn(summary: str) -> CheckpointResult: | |
| return CheckpointResult(approved=True) | |
| service = CoordinatorService(session=session, post_fn=post_fn, checkpoint_fn=chk_fn) | |
| scout._scholar.search = mock_scholar | |
| await service.dispatch( | |
| from_role="human", | |
| text="@scout search for efficient transformers", | |
| ) | |
| state = session.research_state | |
| assert len(state.key_papers) > 0, "Research State must have key papers after Scout" | |
| assert any("Efficient Transformers" in p.title for p in state.key_papers) | |
| async def test_cost_gate_blocks_expensive_dispatch( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| ) -> None: | |
| """When estimated cost >= $0.50, Coordinator asks for approval; rejection cancels the op.""" | |
| scout = ScoutAgent() | |
| session = tmp_sessions.create(goal="Cost gate test") | |
| session.registry.register(RoleEntry(role="scout", agent=scout)) | |
| posts: list[tuple[str, str]] = [] | |
| checkpoints: list[str] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| await asyncio.sleep(0) | |
| async def reject_checkpoint(summary: str) -> CheckpointResult: | |
| checkpoints.append(summary) | |
| await asyncio.sleep(0) | |
| return CheckpointResult(approved=False, response="too expensive") | |
| # Build an LLM service stub that reports a high cost | |
| mock_llm = MagicMock() | |
| mock_llm.estimate_tokens.return_value = 50_000 | |
| mock_llm.calculate_cost.return_value = 1.25 # > $0.50 threshold | |
| from polyhedra.agents.coordinator import CoordinatorService | |
| service = CoordinatorService( | |
| session=session, | |
| post_fn=post_fn, | |
| checkpoint_fn=reject_checkpoint, | |
| llm_service=mock_llm, | |
| ) | |
| scout._scholar.search = mock_scholar | |
| await service.dispatch(from_role="human", text="@scout search for transformers") | |
| # Cost checkpoint must have fired | |
| assert len(checkpoints) == 1, "Cost gate checkpoint must fire" | |
| assert "$1.25" in checkpoints[0] or "cost" in checkpoints[0].lower() | |
| # Scout must NOT have posted (operation was cancelled) | |
| scout_posts = [c for r, c in posts if r == "scout"] | |
| assert len(scout_posts) == 0, "Scout must not run when cost gate is rejected" | |
| # Coordinator must have posted a cancellation notice | |
| coord_posts = [c for r, c in posts if r == "coordinator"] | |
| assert any("cancelled" in c.lower() or "cancel" in c.lower() for c in coord_posts) | |
| async def test_cost_gate_passes_cheap_dispatch( | |
| self, | |
| tmp_sessions: SessionManager, | |
| mock_scholar: MagicMock, | |
| ) -> None: | |
| """When estimated cost < $0.50, Coordinator dispatches without a cost checkpoint.""" | |
| scout = ScoutAgent() | |
| session = tmp_sessions.create(goal="Cheap dispatch test") | |
| session.registry.register(RoleEntry(role="scout", agent=scout)) | |
| posts: list[tuple[str, str]] = [] | |
| checkpoints: list[str] = [] | |
| async def post_fn(role: str, content: str) -> None: | |
| posts.append((role, content)) | |
| await asyncio.sleep(0) | |
| async def chk_fn(summary: str) -> CheckpointResult: | |
| checkpoints.append(summary) | |
| await asyncio.sleep(0) | |
| return CheckpointResult(approved=True) | |
| mock_llm = MagicMock() | |
| mock_llm.estimate_tokens.return_value = 200 | |
| mock_llm.calculate_cost.return_value = 0.01 # well under threshold | |
| from polyhedra.agents.coordinator import CoordinatorService | |
| service = CoordinatorService( | |
| session=session, | |
| post_fn=post_fn, | |
| checkpoint_fn=chk_fn, | |
| llm_service=mock_llm, | |
| ) | |
| scout._scholar.search = mock_scholar | |
| await service.dispatch(from_role="human", text="@scout search for transformers") | |
| # Scout should have run (no cost barrier) | |
| assert any(r == "scout" for r, _ in posts) | |
| # No cost checkpoint (only Scout's own checkpoint may have fired) | |
| # The cost gate checkpoint summary would start with π° | |
| cost_checkpoints = [c for c in checkpoints if c.startswith("π°")] | |
| assert len(cost_checkpoints) == 0, "No cost checkpoint for cheap operations" | |
| async def test_session_export(self, tmp_sessions: SessionManager, tmp_path: Path) -> None: | |
| """Session export produces a zip with expected files.""" | |
| session = tmp_sessions.create(goal="Export test") | |
| session.post("human", "hello") | |
| session.post("scout", "found papers") | |
| session.save_meta() | |
| export_dir = tmp_path / "exports" | |
| zip_path = tmp_sessions.export(session.session_id, export_dir) | |
| assert zip_path.exists() | |
| assert zip_path.suffix == ".zip" | |
| import zipfile | |
| with zipfile.ZipFile(zip_path) as zf: | |
| names = zf.namelist() | |
| assert any("session.json" in n for n in names) | |
| assert any("thread.jsonl" in n for n in names) | |