Muhammad Faraz Malik
feat(llm): add Amazon Bedrock Converse multi-cloud LLM provider alongside Groq
bd6aedd Download tests/unit/test_routing.py from MF180/autonomous-bug-fixing-agent: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/MF180/autonomous-bug-fixing-agent/resolve/main/tests/unit/test_routing.py
- Command line
-
hf download hf://MF180/autonomous-bug-fixing-agent/tests/unit/test_routing.py
-
curl -L -o test_routing.py https://huggingface.co/MF180/autonomous-bug-fixing-agent/resolve/main/tests/unit/test_routing.py
12.6 kB
| """Unit tests for conditional edge routers, `Workspace` file tools, `LocalSandbox`, | |
| and the full production `build_graph()` with an injected fake LLM. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| import pytest | |
| from langchain_core.language_models.chat_models import BaseChatModel | |
| from langchain_core.messages import AIMessage, BaseMessage | |
| from langchain_core.outputs import ChatGeneration, ChatResult | |
| from langgraph.types import Command | |
| from agent.graph import ( | |
| build_graph, | |
| route_after_apply, | |
| route_after_approval, | |
| route_after_repro, | |
| route_after_tests, | |
| ) | |
| from agent.state import ( | |
| AgentState, | |
| AgentStatus, | |
| ApprovalDecision, | |
| FileEdit, | |
| FixAttempt, | |
| IssueInput, | |
| ReproTest, | |
| RunBudget, | |
| SandboxResult, | |
| initial_state, | |
| ) | |
| from sandbox.local_runner import LocalSandbox, scrubbed_env | |
| from tools.file_tools import ( | |
| EditError, | |
| PathEscapeError, | |
| Workspace, | |
| create_repo_from_uploads, | |
| validate_repro_path, | |
| ) | |
| FIXTURE_REPO = Path(__file__).resolve().parents[1] / "fixtures" / "buggy_repo" | |
| class ScriptedChatModel(BaseChatModel): | |
| """Minimal deterministic `BaseChatModel` returning pre-canned JSON responses.""" | |
| responses: list[str] | |
| idx: int = 0 | |
| def _llm_type(self) -> str: | |
| return "scripted" | |
| def _generate( | |
| self, | |
| messages: list[BaseMessage], | |
| stop: list[str] | None = None, | |
| **kwargs: Any, | |
| ) -> ChatResult: | |
| text = self.responses[min(self.idx, len(self.responses) - 1)] | |
| self.idx += 1 | |
| return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text))]) | |
| # ------------------------------------------------------------------ pure routers | |
| def test_route_after_repro_branches(): | |
| base: AgentState = {"budget": RunBudget(max_repro_attempts=2), "repro_attempts": 1} | |
| assert ( | |
| route_after_repro( | |
| {**base, "repro_test": ReproTest(path="test_a.py", content="x", confirmed_failing=True)} | |
| ) | |
| == "propose_fix" | |
| ) | |
| assert ( | |
| route_after_repro( | |
| { | |
| **base, | |
| "repro_test": ReproTest(path="test_a.py", content="x", confirmed_failing=False), | |
| } | |
| ) | |
| == "write_repro_test" | |
| ) | |
| assert ( | |
| route_after_repro( | |
| { | |
| **base, | |
| "repro_attempts": 2, | |
| "repro_test": ReproTest(path="test_a.py", content="x", confirmed_failing=False), | |
| } | |
| ) | |
| == "escalate" | |
| ) | |
| def test_route_after_apply_skips_sandbox_on_patch_error(): | |
| state: AgentState = { | |
| "budget": RunBudget(), | |
| "iteration": 1, | |
| "attempts": [FixAttempt(iteration=1, patch_error="search not found")], | |
| } | |
| assert route_after_apply(state) == "analyze_failure" | |
| state["attempts"] = [FixAttempt(iteration=1, patch_error=None)] | |
| assert route_after_apply(state) == "run_tests" | |
| def test_route_after_tests_respects_budget_and_timeout(): | |
| budget = RunBudget(max_iterations=8) | |
| passing = FixAttempt(iteration=1, result=SandboxResult(exit_code=0, stdout="ok", stderr="")) | |
| failing = FixAttempt(iteration=1, result=SandboxResult(exit_code=1, stdout="F", stderr="")) | |
| timed_out = FixAttempt( | |
| iteration=1, result=SandboxResult(exit_code=0, stdout="", stderr="", timed_out=True) | |
| ) | |
| assert route_after_tests({"budget": budget, "iteration": 1, "attempts": [passing]}) == "human_approval" | |
| assert route_after_tests({"budget": budget, "iteration": 1, "attempts": [failing]}) == "analyze_failure" | |
| assert route_after_tests({"budget": budget, "iteration": 1, "attempts": [timed_out]}) == "analyze_failure" | |
| assert route_after_tests({"budget": budget, "iteration": 8, "attempts": [failing]}) == "escalate" | |
| def test_route_after_approval_branches(): | |
| budget = RunBudget(max_iterations=8) | |
| assert ( | |
| route_after_approval( | |
| {"budget": budget, "iteration": 1, "approval": ApprovalDecision(approved=True)} | |
| ) | |
| == "commit_and_pr" | |
| ) | |
| assert ( | |
| route_after_approval( | |
| { | |
| "budget": budget, | |
| "iteration": 1, | |
| "approval": ApprovalDecision(approved=False, feedback="add guard"), | |
| } | |
| ) | |
| == "analyze_failure" | |
| ) | |
| assert ( | |
| route_after_approval( | |
| {"budget": budget, "iteration": 1, "approval": ApprovalDecision(approved=False)} | |
| ) | |
| == "escalate" | |
| ) | |
| # --------------------------------------------------------- Workspace + LocalSandbox | |
| def test_workspace_jails_paths_and_applies_edits(): | |
| ws = Workspace.create(FIXTURE_REPO) | |
| try: | |
| with pytest.raises(PathEscapeError): | |
| ws.resolve("../outside.py") | |
| with pytest.raises(PathEscapeError): | |
| validate_repro_path("calc.py") | |
| ws.apply_edits( | |
| [ | |
| FileEdit( | |
| path="calc.py", | |
| search="return sum(xs) / (len(xs) + 1)", | |
| replace="return sum(xs) / len(xs)", | |
| ) | |
| ] | |
| ) | |
| assert "+ return sum(xs) / len(xs)" in ws.diff("calc.py") | |
| with pytest.raises(EditError): | |
| ws.apply_edits([FileEdit(path="calc.py", search="nonexistent line", replace="x")]) | |
| ws.reset() | |
| assert ws.diff("calc.py") == "" | |
| zip_bytes = ws.export_zip_bytes() | |
| assert len(zip_bytes) > 100 | |
| finally: | |
| ws.cleanup() | |
| def test_parse_github_owner_repo(): | |
| from tools.git_tools import parse_github_issue_url, parse_github_owner_repo | |
| assert parse_github_owner_repo("https://github.com/farazmalik180/SwiftSupport-Commerce.git") == ( | |
| "farazmalik180", | |
| "SwiftSupport-Commerce", | |
| ) | |
| assert parse_github_owner_repo("https://github.com/owner/repo") == ("owner", "repo") | |
| assert parse_github_owner_repo("not-a-url") is None | |
| assert parse_github_issue_url("https://github.com/farazmalik180/SwiftSupport-Commerce/issues/12") == ( | |
| "farazmalik180", | |
| "SwiftSupport-Commerce", | |
| 12, | |
| ) | |
| assert parse_github_issue_url("https://github.com/owner/repo") is None | |
| def test_aws_tools_s3_and_dynamodb_with_mocks(tmp_path: Path): | |
| from unittest.mock import MagicMock | |
| from tools.aws_tools import ( | |
| list_runs_from_dynamodb, | |
| save_run_to_dynamodb, | |
| upload_bytes_to_s3, | |
| ) | |
| fake_s3 = MagicMock() | |
| fake_s3.generate_presigned_url.return_value = "https://s3.amazonaws.com/bucket/report.pdf" | |
| url = upload_bytes_to_s3( | |
| b"%PDF-1.4", | |
| "reports/run-1/bugfix-report.pdf", | |
| "application/pdf", | |
| bucket="my-bucket", | |
| s3_client=fake_s3, | |
| ) | |
| assert url == "https://s3.amazonaws.com/bucket/report.pdf" | |
| fake_s3.put_object.assert_called_once() | |
| fake_ddb = MagicMock() | |
| fake_ddb.scan.return_value = { | |
| "Items": [ | |
| { | |
| "run_id": {"S": "live-1234"}, | |
| "created_at": {"S": "2026-11-17T18:00:00Z"}, | |
| "repo": {"S": "https://github.com/owner/repo"}, | |
| "issue_title": {"S": "Bug in average()"}, | |
| "status": {"S": "committed"}, | |
| "iterations": {"N": "2"}, | |
| "findings_count": {"N": "3"}, | |
| "s3_pdf_url": {"S": "https://s3.amazonaws.com/bucket/report.pdf"}, | |
| } | |
| ] | |
| } | |
| saved = save_run_to_dynamodb( | |
| "live-1234", | |
| { | |
| "issue": IssueInput(title="Bug in average()", body="desc", repo_path=tmp_path), | |
| "status": AgentStatus.COMMITTED, | |
| "iteration": 2, | |
| "commit_sha": "abc1234", | |
| }, | |
| s3_pdf_url=url, | |
| table_name="BugFixAgentRuns", | |
| ddb_client=fake_ddb, | |
| ) | |
| assert saved is True | |
| fake_ddb.put_item.assert_called_once() | |
| runs = list_runs_from_dynamodb(limit=5, table_name="BugFixAgentRuns", ddb_client=fake_ddb) | |
| assert len(runs) == 1 | |
| assert runs[0]["run_id"] == "live-1234" | |
| assert runs[0]["s3_pdf_url"] == "https://s3.amazonaws.com/bucket/report.pdf" | |
| def test_bedrock_converse_chat_model_with_mock(): | |
| from unittest.mock import MagicMock | |
| from langchain_core.messages import HumanMessage, SystemMessage | |
| from agent.llm import BedrockConverseChatModel, create_groq_llm | |
| llm = create_groq_llm(model="bedrock/us.amazon.nova-lite-v1:0") | |
| assert isinstance(llm, BedrockConverseChatModel) | |
| assert llm.model_id == "us.amazon.nova-lite-v1:0" | |
| fake_brt = MagicMock() | |
| fake_brt.converse.return_value = { | |
| "output": {"message": {"content": [{"text": '{"ok": true}'}]}} | |
| } | |
| llm.client = fake_brt | |
| res = llm.invoke([SystemMessage(content="sys"), HumanMessage(content="hi")]) | |
| assert res.content == '{"ok": true}' | |
| fake_brt.converse.assert_called_once() | |
| def test_create_repo_from_uploads_handles_zip_and_rejects_zip_slip(): | |
| import io | |
| import shutil | |
| import zipfile | |
| buf = io.BytesIO() | |
| with zipfile.ZipFile(buf, "w") as zf: | |
| zf.writestr("myrepo-main/calc.py", "def add(a, b):\n return a - b\n") | |
| zf.writestr("myrepo-main/sub/util.py", "X = 1\n") | |
| repo_dir = create_repo_from_uploads([("myrepo.zip", buf.getvalue())]) | |
| try: | |
| assert (repo_dir / "calc.py").is_file() | |
| assert (repo_dir / "sub" / "util.py").is_file() | |
| finally: | |
| shutil.rmtree(repo_dir, ignore_errors=True) | |
| # Zip-slip attempt must raise PathEscapeError | |
| bad_buf = io.BytesIO() | |
| with zipfile.ZipFile(bad_buf, "w") as zf: | |
| zf.writestr("../../evil.py", "print('pwned')\n") | |
| with pytest.raises(PathEscapeError): | |
| create_repo_from_uploads([("evil.zip", bad_buf.getvalue())]) | |
| def test_local_sandbox_scrubs_secrets_and_runs_pytest(tmp_path: Path): | |
| env = scrubbed_env({"GROQ_API_KEY": "secret", "GITHUB_TOKEN": "tok", "SAFE_VAR": "1"}) | |
| assert "GROQ_API_KEY" not in env and "GITHUB_TOKEN" not in env | |
| assert env["SAFE_VAR"] == "1" | |
| with LocalSandbox(default_timeout_s=15) as sb: | |
| sb.start(tmp_path) | |
| sb.write_file("test_sample.py", "def test_ok():\n assert 2 + 2 == 4\n") | |
| res = sb.run_tests("pytest -q -p no:cacheprovider") | |
| assert res.passed, res.stdout + res.stderr | |
| # ------------------------------------------------- Production graph end-to-end (fake LLM) | |
| def test_production_graph_fixes_bug_and_pauses_at_approval(): | |
| repro_json = json.dumps( | |
| { | |
| "path": "test_repro_bug.py", | |
| "content": ( | |
| "from calc import average\n\n" | |
| "def test_average_repro():\n" | |
| " assert average([2, 4, 6]) == 4.0\n" | |
| ), | |
| } | |
| ) | |
| fix_json = json.dumps( | |
| { | |
| "rationale": "Remove off-by-one +1 in average() denominator.", | |
| "edits": [ | |
| { | |
| "path": "calc.py", | |
| "search": "return sum(xs) / (len(xs) + 1)", | |
| "replace": "return sum(xs) / len(xs)", | |
| } | |
| ], | |
| } | |
| ) | |
| fake_llm = ScriptedChatModel(responses=[repro_json, fix_json]) | |
| graph = build_graph() | |
| cfg = { | |
| "configurable": { | |
| "thread_id": "prod-e2e-1", | |
| "llm": fake_llm, | |
| "sandbox_backend": "local", | |
| "commit_in_git": False, | |
| } | |
| } | |
| issue = IssueInput( | |
| title="average() is off by one", | |
| body="average([2, 4, 6]) returns 3.0 instead of 4.0", | |
| repo_path=FIXTURE_REPO, | |
| ) | |
| nodes = [ | |
| n for chunk in graph.stream(initial_state(issue), cfg, stream_mode="updates") for n in chunk | |
| ] | |
| snap = graph.get_state(cfg) | |
| try: | |
| assert snap.next == ("human_approval",) | |
| assert "commit_and_pr" not in nodes | |
| assert snap.values["repro_test"].confirmed_failing is True | |
| assert snap.values["attempts"][-1].result.passed is True | |
| # Resume with approval | |
| after_nodes = [ | |
| n | |
| for chunk in graph.stream( | |
| Command(resume={"approved": True, "reviewer": "tester"}), | |
| cfg, | |
| stream_mode="updates", | |
| ) | |
| for n in chunk | |
| ] | |
| assert after_nodes[-1] == "commit_and_pr" | |
| final = graph.get_state(cfg).values | |
| assert final["status"] == AgentStatus.COMMITTED | |
| assert final["commit_sha"] is not None | |
| finally: | |
| ws_root = snap.values.get("workspace_root") | |
| if ws_root: | |
| Workspace.open(ws_root).cleanup() | |