PolyHedra / tests /test_services /test_structured_completion.py
yf176
fix: validate macro evidence within structured correction
7d8f4f4
Raw History Blame Contribute Delete
5.84 kB
"""Structured responses must validate before they can leave the LLM boundary."""
from datetime import datetime
from typing import Literal
from unittest.mock import AsyncMock
import pytest
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
from polyhedra.services.llm_service import (
AuthenticationError, ConfigurationError, LLMService, StructuredOutputError,
)
class Finding(BaseModel):
model_config = ConfigDict(extra="forbid")
ticker: Literal["SPY"]
confidence: float = Field(ge=0, le=1)
as_of: AwareDatetime
VALID = '{"ticker":"SPY","confidence":0.5,"as_of":"2026-09-09T00:00:00Z"}'
@pytest.fixture
def service():
service = LLMService(provider="openai", api_key="test-only")
service.complete = AsyncMock(return_value=(VALID, 10, 20))
return service
async def test_valid_output_and_schema_prompt(service):
result = await service.complete_structured("Analyze provided evidence", Finding, model="configured-model")
assert isinstance(result.output, Finding)
assert isinstance(result.output.as_of, datetime)
assert (result.input_tokens, result.output_tokens, result.attempts) == (10, 20, 1)
request, model = service.complete.call_args.args
assert '"additionalProperties": false' in request
assert model == "configured-model"
assert service.complete.call_args.kwargs["response_schema"] == Finding.model_json_schema()
assert service.complete.call_args.kwargs["schema_name"] == "Finding"
async def test_correction_accounts_for_both_attempts(service):
service.complete.side_effect = [(VALID.replace("0.5", "1.5"), 10, 20), (VALID, 30, 40)]
result = await service.complete_structured("Evidence", Finding)
assert (result.input_tokens, result.output_tokens, result.attempts) == (40, 60, 2)
assert "confidence: less_than_equal" in service.complete.call_args.args[0]
assert "Input should be less than or equal to 1" in service.complete.call_args.args[0]
async def test_evidence_validation_uses_existing_correction_attempt(service):
service.complete.side_effect = [(VALID, 10, 20), (VALID, 30, 40)]
checks = 0
def validate(output):
nonlocal checks
checks += 1
if checks == 1:
raise ValueError("evidence JSON Pointer does not exist")
result = await service.complete_structured(
"Evidence", Finding, validate_output=validate,
)
assert result.attempts == 2
assert checks == 2
assert (result.input_tokens, result.output_tokens) == (40, 60)
assert "evidence JSON Pointer does not exist" in service.complete.call_args.args[0]
async def test_evidence_validation_still_blocks_after_correction(service):
def reject(_output):
raise ValueError("evidence JSON Pointer does not exist")
with pytest.raises(StructuredOutputError, match="evidence JSON Pointer does not exist"):
await service.complete_structured("Evidence", Finding, validate_output=reject)
assert service.complete.await_count == 2
async def test_json_parse_error_gives_correction_location_without_response_text(service):
malformed = "```json\n" + VALID + "\n```"
service.complete.side_effect = [(malformed, 10, 20), (VALID, 30, 40)]
result = await service.complete_structured("Evidence", Finding)
assert result.attempts == 2
correction = service.complete.call_args.args[0]
assert "invalid JSON at line 1, column 1" in correction
assert "```json" not in correction
@pytest.mark.parametrize("text", [
"# Analysis\nSPY looks bullish", f"```json\n{VALID}\n```", "{}", "[]",
VALID.replace('"SPY"', '"QQQ"'), VALID.replace("0.5", '"0.5"'),
VALID.replace("0.5", "NaN"), VALID.replace("0.5", "Infinity"),
VALID.replace("0.5", "1e999"),
VALID.replace('"ticker":', '"ticker":"QQQ","ticker":'),
VALID[:-1] + ',"next_role":"human"}', VALID + VALID,
])
async def test_invalid_output_is_never_a_success(service, text):
service.complete.return_value = (text, 10, 20)
with pytest.raises(StructuredOutputError) as caught:
await service.complete_structured("Evidence", Finding)
error = caught.value
assert (error.input_tokens, error.output_tokens, error.attempts) == (20, 40, 2)
assert service.complete.await_count == 2
async def test_caller_can_disable_correction(service):
service.complete.return_value = ("bad", 10, 20)
with pytest.raises(StructuredOutputError) as caught:
await service.complete_structured("Evidence", Finding, max_attempts=1)
assert caught.value.attempts == 1
assert service.complete.await_count == 1
async def test_open_nested_model_is_rejected_before_request(service):
class OpenPayload(BaseModel):
value: str
class Container(BaseModel):
model_config = ConfigDict(extra="forbid")
payload: OpenPayload
with pytest.raises(ValueError, match="forbid extra"):
await service.complete_structured("Evidence", Container)
service.complete.assert_not_called()
async def test_configuration_and_provider_errors_are_not_format_retries(service):
service._configured = False
with pytest.raises(ConfigurationError):
await service.complete_structured("Evidence", Finding)
service.complete.assert_not_called()
service._configured = True
service.complete.side_effect = AuthenticationError("invalid credential")
with pytest.raises(AuthenticationError):
await service.complete_structured("Evidence", Finding)
assert service.complete.await_count == 1
async def test_error_does_not_include_input_values(service):
service.complete.return_value = (VALID.replace('"SPY"', '"private-input"'), 1, 1)
with pytest.raises(StructuredOutputError) as caught:
await service.complete_structured("Evidence", Finding)
assert "private-input" not in str(caught.value)