Download tests/test_services/test_structured_completion.py from polyHedra/PolyHedra: direct link, hf CLI and curl.
- Browser
- Download file 5.84 kB
-
https://huggingface.co/spaces/polyHedra/PolyHedra/resolve/main/tests/test_services/test_structured_completion.py
- Command line
-
hf download hf://spaces/polyHedra/PolyHedra/tests/test_services/test_structured_completion.py
-
curl -L -o test_structured_completion.py https://huggingface.co/spaces/polyHedra/PolyHedra/resolve/main/tests/test_services/test_structured_completion.py
5.84 kB
| """Structured responses must validate before they can leave the LLM boundary.""" | |
| from datetime import datetime | |
| from typing import Literal | |
| from unittest.mock import AsyncMock | |
| import pytest | |
| from pydantic import AwareDatetime, BaseModel, ConfigDict, Field | |
| from polyhedra.services.llm_service import ( | |
| AuthenticationError, ConfigurationError, LLMService, StructuredOutputError, | |
| ) | |
| class Finding(BaseModel): | |
| model_config = ConfigDict(extra="forbid") | |
| ticker: Literal["SPY"] | |
| confidence: float = Field(ge=0, le=1) | |
| as_of: AwareDatetime | |
| VALID = '{"ticker":"SPY","confidence":0.5,"as_of":"2026-09-09T00:00:00Z"}' | |
| def service(): | |
| service = LLMService(provider="openai", api_key="test-only") | |
| service.complete = AsyncMock(return_value=(VALID, 10, 20)) | |
| return service | |
| async def test_valid_output_and_schema_prompt(service): | |
| result = await service.complete_structured("Analyze provided evidence", Finding, model="configured-model") | |
| assert isinstance(result.output, Finding) | |
| assert isinstance(result.output.as_of, datetime) | |
| assert (result.input_tokens, result.output_tokens, result.attempts) == (10, 20, 1) | |
| request, model = service.complete.call_args.args | |
| assert '"additionalProperties": false' in request | |
| assert model == "configured-model" | |
| assert service.complete.call_args.kwargs["response_schema"] == Finding.model_json_schema() | |
| assert service.complete.call_args.kwargs["schema_name"] == "Finding" | |
| async def test_correction_accounts_for_both_attempts(service): | |
| service.complete.side_effect = [(VALID.replace("0.5", "1.5"), 10, 20), (VALID, 30, 40)] | |
| result = await service.complete_structured("Evidence", Finding) | |
| assert (result.input_tokens, result.output_tokens, result.attempts) == (40, 60, 2) | |
| assert "confidence: less_than_equal" in service.complete.call_args.args[0] | |
| assert "Input should be less than or equal to 1" in service.complete.call_args.args[0] | |
| async def test_evidence_validation_uses_existing_correction_attempt(service): | |
| service.complete.side_effect = [(VALID, 10, 20), (VALID, 30, 40)] | |
| checks = 0 | |
| def validate(output): | |
| nonlocal checks | |
| checks += 1 | |
| if checks == 1: | |
| raise ValueError("evidence JSON Pointer does not exist") | |
| result = await service.complete_structured( | |
| "Evidence", Finding, validate_output=validate, | |
| ) | |
| assert result.attempts == 2 | |
| assert checks == 2 | |
| assert (result.input_tokens, result.output_tokens) == (40, 60) | |
| assert "evidence JSON Pointer does not exist" in service.complete.call_args.args[0] | |
| async def test_evidence_validation_still_blocks_after_correction(service): | |
| def reject(_output): | |
| raise ValueError("evidence JSON Pointer does not exist") | |
| with pytest.raises(StructuredOutputError, match="evidence JSON Pointer does not exist"): | |
| await service.complete_structured("Evidence", Finding, validate_output=reject) | |
| assert service.complete.await_count == 2 | |
| async def test_json_parse_error_gives_correction_location_without_response_text(service): | |
| malformed = "```json\n" + VALID + "\n```" | |
| service.complete.side_effect = [(malformed, 10, 20), (VALID, 30, 40)] | |
| result = await service.complete_structured("Evidence", Finding) | |
| assert result.attempts == 2 | |
| correction = service.complete.call_args.args[0] | |
| assert "invalid JSON at line 1, column 1" in correction | |
| assert "```json" not in correction | |
| async def test_invalid_output_is_never_a_success(service, text): | |
| service.complete.return_value = (text, 10, 20) | |
| with pytest.raises(StructuredOutputError) as caught: | |
| await service.complete_structured("Evidence", Finding) | |
| error = caught.value | |
| assert (error.input_tokens, error.output_tokens, error.attempts) == (20, 40, 2) | |
| assert service.complete.await_count == 2 | |
| async def test_caller_can_disable_correction(service): | |
| service.complete.return_value = ("bad", 10, 20) | |
| with pytest.raises(StructuredOutputError) as caught: | |
| await service.complete_structured("Evidence", Finding, max_attempts=1) | |
| assert caught.value.attempts == 1 | |
| assert service.complete.await_count == 1 | |
| async def test_open_nested_model_is_rejected_before_request(service): | |
| class OpenPayload(BaseModel): | |
| value: str | |
| class Container(BaseModel): | |
| model_config = ConfigDict(extra="forbid") | |
| payload: OpenPayload | |
| with pytest.raises(ValueError, match="forbid extra"): | |
| await service.complete_structured("Evidence", Container) | |
| service.complete.assert_not_called() | |
| async def test_configuration_and_provider_errors_are_not_format_retries(service): | |
| service._configured = False | |
| with pytest.raises(ConfigurationError): | |
| await service.complete_structured("Evidence", Finding) | |
| service.complete.assert_not_called() | |
| service._configured = True | |
| service.complete.side_effect = AuthenticationError("invalid credential") | |
| with pytest.raises(AuthenticationError): | |
| await service.complete_structured("Evidence", Finding) | |
| assert service.complete.await_count == 1 | |
| async def test_error_does_not_include_input_values(service): | |
| service.complete.return_value = (VALID.replace('"SPY"', '"private-input"'), 1, 1) | |
| with pytest.raises(StructuredOutputError) as caught: | |
| await service.complete_structured("Evidence", Finding) | |
| assert "private-input" not in str(caught.value) | |