File size: 5,836 Bytes
f816c35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b9adcdf
 
f816c35
 
 
 
 
 
 
589695d
f816c35
 
7d8f4f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
053707d
 
 
 
 
 
 
 
 
 
 
 
f816c35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""Structured responses must validate before they can leave the LLM boundary."""

from datetime import datetime
from typing import Literal
from unittest.mock import AsyncMock

import pytest
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field

from polyhedra.services.llm_service import (
    AuthenticationError, ConfigurationError, LLMService, StructuredOutputError,
)


class Finding(BaseModel):
    model_config = ConfigDict(extra="forbid")
    ticker: Literal["SPY"]
    confidence: float = Field(ge=0, le=1)
    as_of: AwareDatetime


VALID = '{"ticker":"SPY","confidence":0.5,"as_of":"2026-09-09T00:00:00Z"}'


@pytest.fixture
def service():
    service = LLMService(provider="openai", api_key="test-only")
    service.complete = AsyncMock(return_value=(VALID, 10, 20))
    return service


async def test_valid_output_and_schema_prompt(service):
    result = await service.complete_structured("Analyze provided evidence", Finding, model="configured-model")
    assert isinstance(result.output, Finding)
    assert isinstance(result.output.as_of, datetime)
    assert (result.input_tokens, result.output_tokens, result.attempts) == (10, 20, 1)
    request, model = service.complete.call_args.args
    assert '"additionalProperties": false' in request
    assert model == "configured-model"
    assert service.complete.call_args.kwargs["response_schema"] == Finding.model_json_schema()
    assert service.complete.call_args.kwargs["schema_name"] == "Finding"


async def test_correction_accounts_for_both_attempts(service):
    service.complete.side_effect = [(VALID.replace("0.5", "1.5"), 10, 20), (VALID, 30, 40)]
    result = await service.complete_structured("Evidence", Finding)
    assert (result.input_tokens, result.output_tokens, result.attempts) == (40, 60, 2)
    assert "confidence: less_than_equal" in service.complete.call_args.args[0]
    assert "Input should be less than or equal to 1" in service.complete.call_args.args[0]


async def test_evidence_validation_uses_existing_correction_attempt(service):
    service.complete.side_effect = [(VALID, 10, 20), (VALID, 30, 40)]
    checks = 0

    def validate(output):
        nonlocal checks
        checks += 1
        if checks == 1:
            raise ValueError("evidence JSON Pointer does not exist")

    result = await service.complete_structured(
        "Evidence", Finding, validate_output=validate,
    )

    assert result.attempts == 2
    assert checks == 2
    assert (result.input_tokens, result.output_tokens) == (40, 60)
    assert "evidence JSON Pointer does not exist" in service.complete.call_args.args[0]


async def test_evidence_validation_still_blocks_after_correction(service):
    def reject(_output):
        raise ValueError("evidence JSON Pointer does not exist")

    with pytest.raises(StructuredOutputError, match="evidence JSON Pointer does not exist"):
        await service.complete_structured("Evidence", Finding, validate_output=reject)

    assert service.complete.await_count == 2


async def test_json_parse_error_gives_correction_location_without_response_text(service):
    malformed = "```json\n" + VALID + "\n```"
    service.complete.side_effect = [(malformed, 10, 20), (VALID, 30, 40)]

    result = await service.complete_structured("Evidence", Finding)

    assert result.attempts == 2
    correction = service.complete.call_args.args[0]
    assert "invalid JSON at line 1, column 1" in correction
    assert "```json" not in correction


@pytest.mark.parametrize("text", [
    "# Analysis\nSPY looks bullish", f"```json\n{VALID}\n```", "{}", "[]",
    VALID.replace('"SPY"', '"QQQ"'), VALID.replace("0.5", '"0.5"'),
    VALID.replace("0.5", "NaN"), VALID.replace("0.5", "Infinity"),
    VALID.replace("0.5", "1e999"),
    VALID.replace('"ticker":', '"ticker":"QQQ","ticker":'),
    VALID[:-1] + ',"next_role":"human"}', VALID + VALID,
])
async def test_invalid_output_is_never_a_success(service, text):
    service.complete.return_value = (text, 10, 20)
    with pytest.raises(StructuredOutputError) as caught:
        await service.complete_structured("Evidence", Finding)
    error = caught.value
    assert (error.input_tokens, error.output_tokens, error.attempts) == (20, 40, 2)
    assert service.complete.await_count == 2


async def test_caller_can_disable_correction(service):
    service.complete.return_value = ("bad", 10, 20)
    with pytest.raises(StructuredOutputError) as caught:
        await service.complete_structured("Evidence", Finding, max_attempts=1)
    assert caught.value.attempts == 1
    assert service.complete.await_count == 1


async def test_open_nested_model_is_rejected_before_request(service):
    class OpenPayload(BaseModel):
        value: str

    class Container(BaseModel):
        model_config = ConfigDict(extra="forbid")
        payload: OpenPayload

    with pytest.raises(ValueError, match="forbid extra"):
        await service.complete_structured("Evidence", Container)
    service.complete.assert_not_called()


async def test_configuration_and_provider_errors_are_not_format_retries(service):
    service._configured = False
    with pytest.raises(ConfigurationError):
        await service.complete_structured("Evidence", Finding)
    service.complete.assert_not_called()
    service._configured = True
    service.complete.side_effect = AuthenticationError("invalid credential")
    with pytest.raises(AuthenticationError):
        await service.complete_structured("Evidence", Finding)
    assert service.complete.await_count == 1


async def test_error_does_not_include_input_values(service):
    service.complete.return_value = (VALID.replace('"SPY"', '"private-input"'), 1, 1)
    with pytest.raises(StructuredOutputError) as caught:
        await service.complete_structured("Evidence", Finding)
    assert "private-input" not in str(caught.value)