"""Structured responses must validate before they can leave the LLM boundary.""" from datetime import datetime from typing import Literal from unittest.mock import AsyncMock import pytest from pydantic import AwareDatetime, BaseModel, ConfigDict, Field from polyhedra.services.llm_service import ( AuthenticationError, ConfigurationError, LLMService, StructuredOutputError, ) class Finding(BaseModel): model_config = ConfigDict(extra="forbid") ticker: Literal["SPY"] confidence: float = Field(ge=0, le=1) as_of: AwareDatetime VALID = '{"ticker":"SPY","confidence":0.5,"as_of":"2026-09-09T00:00:00Z"}' @pytest.fixture def service(): service = LLMService(provider="openai", api_key="test-only") service.complete = AsyncMock(return_value=(VALID, 10, 20)) return service async def test_valid_output_and_schema_prompt(service): result = await service.complete_structured("Analyze provided evidence", Finding, model="configured-model") assert isinstance(result.output, Finding) assert isinstance(result.output.as_of, datetime) assert (result.input_tokens, result.output_tokens, result.attempts) == (10, 20, 1) request, model = service.complete.call_args.args assert '"additionalProperties": false' in request assert model == "configured-model" assert service.complete.call_args.kwargs["response_schema"] == Finding.model_json_schema() assert service.complete.call_args.kwargs["schema_name"] == "Finding" async def test_correction_accounts_for_both_attempts(service): service.complete.side_effect = [(VALID.replace("0.5", "1.5"), 10, 20), (VALID, 30, 40)] result = await service.complete_structured("Evidence", Finding) assert (result.input_tokens, result.output_tokens, result.attempts) == (40, 60, 2) assert "confidence: less_than_equal" in service.complete.call_args.args[0] assert "Input should be less than or equal to 1" in service.complete.call_args.args[0] async def test_evidence_validation_uses_existing_correction_attempt(service): service.complete.side_effect = [(VALID, 10, 20), (VALID, 30, 40)] checks = 0 def validate(output): nonlocal checks checks += 1 if checks == 1: raise ValueError("evidence JSON Pointer does not exist") result = await service.complete_structured( "Evidence", Finding, validate_output=validate, ) assert result.attempts == 2 assert checks == 2 assert (result.input_tokens, result.output_tokens) == (40, 60) assert "evidence JSON Pointer does not exist" in service.complete.call_args.args[0] async def test_evidence_validation_still_blocks_after_correction(service): def reject(_output): raise ValueError("evidence JSON Pointer does not exist") with pytest.raises(StructuredOutputError, match="evidence JSON Pointer does not exist"): await service.complete_structured("Evidence", Finding, validate_output=reject) assert service.complete.await_count == 2 async def test_json_parse_error_gives_correction_location_without_response_text(service): malformed = "```json\n" + VALID + "\n```" service.complete.side_effect = [(malformed, 10, 20), (VALID, 30, 40)] result = await service.complete_structured("Evidence", Finding) assert result.attempts == 2 correction = service.complete.call_args.args[0] assert "invalid JSON at line 1, column 1" in correction assert "```json" not in correction @pytest.mark.parametrize("text", [ "# Analysis\nSPY looks bullish", f"```json\n{VALID}\n```", "{}", "[]", VALID.replace('"SPY"', '"QQQ"'), VALID.replace("0.5", '"0.5"'), VALID.replace("0.5", "NaN"), VALID.replace("0.5", "Infinity"), VALID.replace("0.5", "1e999"), VALID.replace('"ticker":', '"ticker":"QQQ","ticker":'), VALID[:-1] + ',"next_role":"human"}', VALID + VALID, ]) async def test_invalid_output_is_never_a_success(service, text): service.complete.return_value = (text, 10, 20) with pytest.raises(StructuredOutputError) as caught: await service.complete_structured("Evidence", Finding) error = caught.value assert (error.input_tokens, error.output_tokens, error.attempts) == (20, 40, 2) assert service.complete.await_count == 2 async def test_caller_can_disable_correction(service): service.complete.return_value = ("bad", 10, 20) with pytest.raises(StructuredOutputError) as caught: await service.complete_structured("Evidence", Finding, max_attempts=1) assert caught.value.attempts == 1 assert service.complete.await_count == 1 async def test_open_nested_model_is_rejected_before_request(service): class OpenPayload(BaseModel): value: str class Container(BaseModel): model_config = ConfigDict(extra="forbid") payload: OpenPayload with pytest.raises(ValueError, match="forbid extra"): await service.complete_structured("Evidence", Container) service.complete.assert_not_called() async def test_configuration_and_provider_errors_are_not_format_retries(service): service._configured = False with pytest.raises(ConfigurationError): await service.complete_structured("Evidence", Finding) service.complete.assert_not_called() service._configured = True service.complete.side_effect = AuthenticationError("invalid credential") with pytest.raises(AuthenticationError): await service.complete_structured("Evidence", Finding) assert service.complete.await_count == 1 async def test_error_does_not_include_input_values(service): service.complete.return_value = (VALID.replace('"SPY"', '"private-input"'), 1, 1) with pytest.raises(StructuredOutputError) as caught: await service.complete_structured("Evidence", Finding) assert "private-input" not in str(caught.value)