Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /als-apg_osprey /tests /cli /test_audit_cmd.py
| """Tests for the osprey audit command.""" | |
| from __future__ import annotations | |
| import json | |
| from unittest.mock import MagicMock, patch | |
| import click | |
| import pytest | |
| from click.testing import CliRunner | |
| from osprey.cli.audit_cmd import _detect_target_type, _extract_json, _list_files | |
| from osprey.cli.audit_prompts import AuditFinding, AuditReport | |
| # --------------------------------------------------------------------------- | |
| # Fixtures | |
| # --------------------------------------------------------------------------- | |
| def runner(): | |
| return CliRunner() | |
| def sample_report() -> AuditReport: | |
| return AuditReport( | |
| summary="Test audit complete", | |
| overall_risk="low", | |
| findings=[ | |
| AuditFinding( | |
| category="permissions", | |
| severity="warning", | |
| title="Open permission", | |
| explanation="A permission is too broad", | |
| file_path="settings.json", | |
| recommendation="Restrict the permission scope", | |
| ), | |
| ], | |
| ) | |
| def sample_report_json(sample_report) -> str: | |
| return sample_report.model_dump_json() | |
| def tmp_project(tmp_path): | |
| """Create a minimal project directory.""" | |
| (tmp_path / "config.yml").write_text("name: test\n") | |
| (tmp_path / ".claude" / "settings.json").mkdir(parents=True, exist_ok=True) | |
| (tmp_path / ".claude" / "settings.json").rmdir() | |
| (tmp_path / ".claude").rmdir() | |
| subdir = tmp_path / "hooks" | |
| subdir.mkdir() | |
| (subdir / "pre_write.sh").write_text("#!/bin/bash\n") | |
| return tmp_path | |
| def tmp_profile(tmp_path): | |
| """Create a minimal profile YAML.""" | |
| profile = tmp_path / "test-profile.yml" | |
| profile.write_text("name: test\nprovider: mock\nmodel: test\n") | |
| return profile | |
| # --------------------------------------------------------------------------- | |
| # Input detection tests | |
| # --------------------------------------------------------------------------- | |
| class TestDetectTargetType: | |
| def test_detect_yaml_profile(self, tmp_profile): | |
| assert _detect_target_type(str(tmp_profile)) == "profile" | |
| def test_detect_yaml_extension(self, tmp_path): | |
| f = tmp_path / "profile.yaml" | |
| f.write_text("name: test\n") | |
| assert _detect_target_type(str(f)) == "profile" | |
| def test_detect_directory(self, tmp_project): | |
| assert _detect_target_type(str(tmp_project)) == "project" | |
| def test_invalid_target(self, tmp_path): | |
| f = tmp_path / "readme.txt" | |
| f.write_text("hello") | |
| with pytest.raises(click.BadParameter, match="must be a .yml/.yaml"): | |
| _detect_target_type(str(f)) | |
| # --------------------------------------------------------------------------- | |
| # File listing tests | |
| # --------------------------------------------------------------------------- | |
| class TestListFiles: | |
| def test_lists_files(self, tmp_project): | |
| listing = _list_files(tmp_project) | |
| assert "config.yml" in listing | |
| assert "hooks/pre_write.sh" in listing | |
| def test_max_files_limit(self, tmp_path): | |
| for i in range(10): | |
| (tmp_path / f"file{i}.txt").write_text("") | |
| listing = _list_files(tmp_path, max_files=3) | |
| assert "... and 7 more files" in listing | |
| # --------------------------------------------------------------------------- | |
| # JSON extraction tests | |
| # --------------------------------------------------------------------------- | |
| class TestExtractJson: | |
| def test_extract_raw_json(self, sample_report_json): | |
| text = f"Here is the report: {sample_report_json}" | |
| result = _extract_json(text) | |
| assert result is not None | |
| parsed = json.loads(result) | |
| assert parsed["overall_risk"] == "low" | |
| def test_extract_markdown_fenced(self, sample_report_json): | |
| text = f"```json\n{sample_report_json}\n```" | |
| result = _extract_json(text) | |
| assert result is not None | |
| parsed = json.loads(result) | |
| assert parsed["overall_risk"] == "low" | |
| def test_extract_no_fence_label(self, sample_report_json): | |
| text = f"```\n{sample_report_json}\n```" | |
| result = _extract_json(text) | |
| assert result is not None | |
| def test_invalid_no_json(self): | |
| assert _extract_json("No JSON here at all") is None | |
| def test_partial_json(self): | |
| # Truncated output — still extracts what's there | |
| result = _extract_json('{"summary": "test", "overall_risk": "low"}') | |
| assert result is not None | |
| # --------------------------------------------------------------------------- | |
| # Pydantic model tests | |
| # --------------------------------------------------------------------------- | |
| class TestAuditModels: | |
| def test_finding_roundtrip(self): | |
| f = AuditFinding( | |
| category="safety", | |
| severity="error", | |
| title="Missing limits hook", | |
| explanation="channel_write lacks bounds checking", | |
| file_path="hooks/pre_write.sh", | |
| recommendation="Add limits hook", | |
| ) | |
| data = json.loads(f.model_dump_json()) | |
| f2 = AuditFinding.model_validate(data) | |
| assert f2.category == "safety" | |
| def test_report_roundtrip(self, sample_report): | |
| data = json.loads(sample_report.model_dump_json()) | |
| r2 = AuditReport.model_validate(data) | |
| assert r2.overall_risk == "low" | |
| assert len(r2.findings) == 1 | |
| def test_report_empty_findings(self): | |
| r = AuditReport(summary="Clean", overall_risk="low", findings=[]) | |
| assert r.findings == [] | |
| # --------------------------------------------------------------------------- | |
| # CLI invocation tests (mock SDK) | |
| # --------------------------------------------------------------------------- | |
| def _make_mock_query(report_json: str): | |
| """Create a mock async generator that yields an AssistantMessage with report JSON.""" | |
| async def mock_query(prompt, options): | |
| msg = MagicMock() | |
| msg.__class__.__name__ = "AssistantMessage" | |
| # Make isinstance check work | |
| text_block = MagicMock() | |
| text_block.text = report_json | |
| msg.content = [text_block] | |
| yield msg | |
| return mock_query | |
| class TestAuditCLI: | |
| """Test CLI invocation with mocked SDK.""" | |
| def _get_audit_cmd(self): | |
| from osprey.cli.audit_cmd import audit | |
| return audit | |
| def test_audit_project_success(self, mock_asyncio, runner, tmp_project, sample_report): | |
| report_json = sample_report.model_dump_json() | |
| mock_asyncio.run.return_value = (report_json, 0.01, 5) | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project)]) | |
| assert result.exit_code == 0 | |
| def test_audit_json_output(self, mock_asyncio, runner, tmp_project, sample_report): | |
| report_json = sample_report.model_dump_json() | |
| mock_asyncio.run.return_value = (report_json, 0.01, 5) | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project), "--json"]) | |
| assert result.exit_code == 0 | |
| output = json.loads(result.output) | |
| assert output["overall_risk"] == "low" | |
| def test_audit_verbose(self, mock_asyncio, runner, tmp_project, sample_report): | |
| report_json = sample_report.model_dump_json() | |
| mock_asyncio.run.return_value = (report_json, 0.05, 10) | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project), "-v"]) | |
| assert result.exit_code == 0 | |
| def test_audit_missing_sdk(self, runner, tmp_project): | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project)]) | |
| assert result.exit_code == 1 | |
| assert "claude-agent-sdk" in result.output | |
| def test_audit_nonexistent_target(self, runner): | |
| result = runner.invoke(self._get_audit_cmd(), ["/nonexistent/path"]) | |
| assert result.exit_code == 2 | |
| def test_audit_invalid_json_output(self, mock_asyncio, runner, tmp_project): | |
| mock_asyncio.run.return_value = ("Not valid JSON at all", None, None) | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project)]) | |
| assert result.exit_code == 1 | |
| def test_audit_markdown_fenced_json(self, mock_asyncio, runner, tmp_project, sample_report): | |
| report_json = sample_report.model_dump_json() | |
| fenced = f"```json\n{report_json}\n```" | |
| mock_asyncio.run.return_value = (fenced, 0.01, 5) | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project)]) | |
| assert result.exit_code == 0 | |
| # --------------------------------------------------------------------------- | |
| # Build flag tests | |
| # --------------------------------------------------------------------------- | |
| class TestBuildFlag: | |
| def _get_audit_cmd(self): | |
| from osprey.cli.audit_cmd import audit | |
| return audit | |
| def test_build_flag_requires_profile(self, runner, tmp_project): | |
| result = runner.invoke(self._get_audit_cmd(), [str(tmp_project), "--build"]) | |
| assert result.exit_code == 1 | |
| # Re-pinned for the renderer's failure shape: summary, cause, remedy. | |
| assert "--build needs a .yml or .yaml profile" in result.output | |
| def test_build_flag_invokes_build_cmd( | |
| self, mock_ctx, mock_asyncio, runner, tmp_profile, sample_report | |
| ): | |
| report_json = sample_report.model_dump_json() | |
| mock_asyncio.run.return_value = (report_json, 0.01, 5) | |
| mock_context = MagicMock() | |
| mock_ctx.return_value = mock_context | |
| runner.invoke(self._get_audit_cmd(), [str(tmp_profile), "--build"]) | |
| # The build invoke is called on the context | |
| assert mock_context.invoke.called | |
| call_kwargs = mock_context.invoke.call_args | |
| # project_name should start with "audit-" | |
| args = call_kwargs[1] if call_kwargs[1] else {} | |
| if "project_name" in args: | |
| assert args["project_name"].startswith("audit-") | |
| # --------------------------------------------------------------------------- | |
| # Display tests | |
| # --------------------------------------------------------------------------- | |
| class TestDisplay: | |
| def test_display_error_finding(self, sample_report, capsys): | |
| from osprey.cli.audit_cmd import _display_report | |
| report = AuditReport( | |
| summary="Issues found", | |
| overall_risk="high", | |
| findings=[ | |
| AuditFinding( | |
| category="safety", | |
| severity="error", | |
| title="Critical issue", | |
| explanation="Details", | |
| file_path="hooks.sh", | |
| recommendation="Fix it", | |
| ), | |
| ], | |
| ) | |
| # Just verify it doesn't crash — Rich output goes to console, not capsys | |
| _display_report(report, json_output=False, verbose=False) | |
| def test_display_json_output(self, sample_report, capsys): | |
| from osprey.cli.audit_cmd import _display_report | |
| _display_report(sample_report, json_output=True, verbose=False) | |
| captured = capsys.readouterr() | |
| parsed = json.loads(captured.out) | |
| assert parsed["overall_risk"] == "low" | |
| def test_display_empty_findings(self): | |
| from osprey.cli.audit_cmd import _display_report | |
| report = AuditReport(summary="Clean", overall_risk="low", findings=[]) | |
| _display_report(report, json_output=False, verbose=False) | |
| def test_display_verbose_with_cost(self, sample_report): | |
| from osprey.cli.audit_cmd import _display_report | |
| _display_report( | |
| sample_report, | |
| json_output=False, | |
| verbose=True, | |
| cost=0.05, | |
| turns=10, | |
| ) | |
Xet Storage Details
- Size:
- 12.4 kB
- Xet hash:
- 39d231463738c25623fd32a276857cdac42cf713b48a630c1f688721860ad280
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.