import json import os import stat import sys import textwrap from dataclasses import replace import pytest from ros_fixtures import INTERFACES, NODE_PY, as_file_blocks, cpp_package, python_package from api.codegen import ( DirectBackend, OpenCodeBackend, generate_code, mentioned_interfaces, nested_types, parse_file_blocks, retrieve, ) from api.llm import LLMError, ScriptedLLM from api.opencode_runner import OpenCodeWorkspace from api.ros_check import validate_package class TestValidatePackage: def test_valid_python_package(self): v = validate_package(python_package(), "humble", INTERFACES) assert v.valid, v.errors assert v.stats["packages"] == ["safety_stop"] assert v.stats["interfaces_used"] == ["geometry_msgs/msg/Twist", "sensor_msgs/msg/LaserScan"] assert v.warnings == [] def test_valid_cpp_package(self): v = validate_package(cpp_package(), "jazzy", INTERFACES) assert v.valid, v.errors assert v.stats["interfaces_used"] == ["std_msgs/msg/String"] def test_hallucinated_message_type(self): node = NODE_PY.replace("from sensor_msgs.msg import LaserScan", "from sensor_msgs.msg import Lidar") v = validate_package(python_package(node=node), "humble", INTERFACES) assert not v.valid assert any("sensor_msgs/msg/Lidar does not exist in ROS 2 humble" in e for e in v.errors) def test_hallucinated_cpp_include(self): files = cpp_package() files["talker_cpp/src/talker.cpp"] = files["talker_cpp/src/talker.cpp"].replace( "std_msgs/msg/string.hpp", "std_msgs/msg/text_line.hpp" ) v = validate_package(files, "humble", INTERFACES) assert any("std_msgs/msg/TextLine does not exist" in e for e in v.errors) def test_unindexed_package_is_a_warning(self): node = NODE_PY.replace("import rclpy\n", "import rclpy\nfrom vision_msgs.msg import Detection2DArray\n") v = validate_package(python_package(node=node), "humble", INTERFACES) assert v.valid and any( "vision_msgs/msg/Detection2DArray is from a package that is not indexed" in w for w in v.warnings ) def test_missing_dependency_warning(self): v = validate_package(python_package(deps=("rclpy",)), "humble", INTERFACES) assert v.valid assert any("uses geometry_msgs but does not declare it" in w for w in v.warnings) def test_syntax_error(self): v = validate_package(python_package(node="def broken(:\n"), "humble", INTERFACES) assert any("Python syntax error" in e for e in v.errors) def test_launch_without_generate_function(self): v = validate_package(python_package(launch="from launch import LaunchDescription\n"), "humble", INTERFACES) assert any("generate_launch_description" in e for e in v.errors) def test_bad_xml_launch(self): files = cpp_package() files["talker_cpp/launch/talker.launch.xml"] = "" assert any("root must be " in e for e in validate_package(files, "humble", INTERFACES).errors) def test_structure_errors(self): files = python_package() del files["safety_stop/setup.py"] assert any("without setup.py" in e for e in validate_package(files, "humble", INTERFACES).errors) files = cpp_package() files["talker_cpp/CMakeLists.txt"] = "project(x)\n" assert any("ament_package()" in e for e in validate_package(files, "humble", INTERFACES).errors) assert validate_package({"a.py": "x = 1"}, "humble", INTERFACES).errors == ["no package.xml found"] assert not validate_package({}, "humble", INTERFACES).valid @pytest.mark.parametrize("bad", ["/etc/cron.d/x", "../escape.py", "pkg/../../x"]) def test_path_traversal(self, bad): v = validate_package({bad: "x", **python_package()}, "humble", INTERFACES) assert not v.valid and "must be relative" in v.errors[0] def test_custom_interfaces_in_package(self): files = python_package() files["safety_stop/msg/Obstacle.msg"] = "float32 distance\n" files["safety_stop/safety_stop/node.py"] += "\nfrom safety_stop.msg import Obstacle\n" assert validate_package(files, "humble", INTERFACES).valid def test_without_catalogue(self): v = validate_package(python_package(), "humble", {}) assert v.valid and any("no interface catalogue" in w for w in v.warnings) class TestRetrieval: def test_mentioned_interfaces(self): found = mentioned_interfaces("subscribe to LaserScan and publish geometry_msgs/Twist", INTERFACES["humble"]) assert found == ["geometry_msgs/msg/Twist", "sensor_msgs/msg/LaserScan"] assert mentioned_interfaces("nothing relevant here", INTERFACES["humble"]) == [] def test_nested_types(self): h = INTERFACES["humble"] assert nested_types(h["geometry_msgs/msg/Twist"], "geometry_msgs", h) == [ "geometry_msgs/msg/Vector3", "geometry_msgs/msg/Vector3", ] assert nested_types(h["sensor_msgs/msg/LaserScan"], "sensor_msgs", h) == ["std_msgs/msg/Header"] def test_retrieve_puts_exact_definitions_first(self, store): hits = retrieve( store, "stop when LaserScan shows an obstacle, publish Twist", "humble", "python", INTERFACES["humble"] ) sources = [h.source for h in hits] assert sources[:4] == [ "interface:humble:geometry_msgs/msg/Twist", "interface:humble:sensor_msgs/msg/LaserScan", "interface:humble:geometry_msgs/msg/Vector3", "interface:humble:std_msgs/msg/Header", ] assert "docs:pubsub" in sources def test_parse_file_blocks(): files = python_package() assert parse_file_blocks("Here you go:\n\n" + as_file_blocks(files)) == {k: v + "\n" for k, v in files.items()} @pytest.fixture def ros_settings(settings): settings.data_dir.mkdir(parents=True, exist_ok=True) settings.interfaces_file.write_text(json.dumps(INTERFACES)) from api.ros_check import load_interfaces load_interfaces.cache_clear() return settings class TestGenerateCodeDirect: def test_repair_loop(self, store, ros_settings): bad = python_package(node=NODE_PY.replace("import LaserScan", "import Lidar")) llm = ScriptedLLM([as_file_blocks(bad), as_file_blocks(python_package())]) result = generate_code( "stop the robot when LaserScan sees an obstacle, publish Twist", "humble", "python", DirectBackend(llm), store, ros_settings, ) assert result.valid and len(result.attempts) == 2 and result.backend == "direct" first_user = llm.calls[0][1].content assert "[ROS 2 humble Message sensor_msgs/msg/LaserScan]" in first_user # exact definition, grounded assert "sensor_msgs/msg/Lidar does not exist" in llm.calls[1][-1].content assert result.files["safety_stop/safety_stop/node.py"].strip() == NODE_PY.strip() def test_invalid_after_all_attempts(self, store, ros_settings): llm = ScriptedLLM(["no files", "still no files"]) result = generate_code( "a publisher of String messages", "humble", "python", DirectBackend(llm), store, ros_settings, max_attempts=2, ) assert not result.valid and result.validation.errors == ["no files were generated"] def test_bad_arguments(self, store, ros_settings): with pytest.raises(ValueError): generate_code("x" * 20, "rolling", "python", DirectBackend(ScriptedLLM([])), store, ros_settings) FAKE_OPENCODE = textwrap.dedent('''\ #!{python} """Stand-in for `opencode run`: writes a package into --dir, emits JSON events.""" import json, os, sys sys.path.insert(0, {tests!r}) from ros_fixtures import NODE_PY, python_package args = sys.argv[1:] ws = args[args.index("--dir") + 1] context = open(args[args.index("-f") + 1]).read() state = os.environ["XDG_DATA_HOME"] assert json.loads(os.environ["OPENCODE_CONFIG_CONTENT"])["permission"]["bash"] == "deny" log = os.path.join(state, "calls.jsonl") with open(log, "a") as f: f.write(json.dumps({{"argv": args, "context": context}}) + "\\n") if os.environ.get("FAKE_OPENCODE_FAIL"): print(json.dumps({{"type": "error", "error": {{"name": "APIError", "data": {{"message": "quota exceeded"}}}}}})) sys.exit(1) continuing = "--continue" in args node = NODE_PY if continuing else NODE_PY.replace("import LaserScan", "import Lidar") for path, content in python_package(node=node).items(): full = os.path.join(ws, path) os.makedirs(os.path.dirname(full), exist_ok=True) open(full, "w").write(content) print(json.dumps({{"type": "text", "part": {{"text": "fixed it" if continuing else "wrote the package"}}}})) ''') @pytest.fixture def fake_opencode(tmp_path, ros_settings): path = tmp_path / "opencode" path.write_text(FAKE_OPENCODE.format(python=sys.executable, tests=os.path.dirname(__file__))) path.chmod(path.stat().st_mode | stat.S_IEXEC) return replace(ros_settings, opencode_bin=str(path), codegen_backend="opencode") class TestOpenCodeBackend: def test_generate_then_repair_in_same_session(self, store, fake_opencode): with OpenCodeWorkspace(fake_opencode) as ws: result = generate_code( "stop the robot when LaserScan sees an obstacle", "humble", "python", OpenCodeBackend(ws), store, fake_opencode, ) calls = [json.loads(line) for line in (ws.state / "calls.jsonl").read_text().splitlines()] root = ws.root assert not root.exists() # workspace cleaned up assert result.valid and result.backend == "opencode" and len(result.attempts) == 2 assert "--continue" not in calls[0]["argv"] and "--continue" in calls[1]["argv"] assert "sensor_msgs/msg/LaserScan" in calls[0]["context"] assert "sensor_msgs/msg/Lidar does not exist" in calls[1]["argv"][1] assert result.attempts[1].transcript == "fixed it" assert "safety_stop/safety_stop/node.py" in result.files def test_opencode_error_surfaces(self, store, fake_opencode, monkeypatch): monkeypatch.setenv("FAKE_OPENCODE_FAIL", "1") with OpenCodeWorkspace(fake_opencode) as ws, pytest.raises(LLMError, match="quota exceeded"): generate_code( "a publisher of String messages", "humble", "python", OpenCodeBackend(ws), store, fake_opencode ) def test_missing_binary(self, store, ros_settings): s = replace(ros_settings, opencode_bin="/nonexistent/opencode") with OpenCodeWorkspace(s) as ws, pytest.raises(LLMError, match="not found"): generate_code("a publisher of String messages", "humble", "python", OpenCodeBackend(ws), store, s)