File size: 8,305 Bytes
115b94d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
import json
import unittest
from concurrent.futures import ThreadPoolExecutor
import httpx
from src.adapters.llm.chat_completion import ChatCompletionProvider
from src.modules.llm_gateway.service import LLMGateway, ModelReply, ProviderFailure
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup, workspace, USER, ModelFixture


class LLMTests(unittest.TestCase):
    def setUp(self):
        self.runtime, request, self.model, _ = setup(self.enterContext(workspace()))
        self.run = self.runtime.create(USER, request, 'model-test-key')
        self.runtime.store.mutate(self.run['id'], lambda r: r.update(status='running', claim='test'))

    def test_usage_settled_and_prompt_not_in_ledger(self):
        result = self.runtime.llm.generate(self.run['id'], 'design', {'requirement': 'private requirement'}, lambda: False)
        self.assertIn('summary', result)
        calls = self.runtime.store.calls(self.run['id'])
        self.assertEqual(calls[0]['charged'], 160)
        self.assertNotIn('private requirement', json.dumps(calls))

    def test_quota_fallback_counts_both_attempts(self):
        first = ModelFixture()
        first.name = 'quota'
        def quota():
            raise ProviderFailure('MODEL_QUOTA_REJECTED', rejected=True, retryable=True)
        first.hook = quota
        gateway = LLMGateway(self.runtime.store, [first, self.model])
        gateway.generate(self.run['id'], 'design', {}, lambda: False)
        self.assertEqual([c['state'] for c in self.runtime.store.calls(self.run['id'])], ['rejected', 'settled'])

    def test_unknown_does_not_fallback(self):
        first = ModelFixture()
        def unknown():
            raise ProviderFailure()
        first.hook = unknown
        with self.assertRaises(SkillError):
            LLMGateway(self.runtime.store, [first, self.model]).generate(self.run['id'], 'design', {}, lambda: False)
        self.assertEqual(self.model.count, 0)

    def test_model_count_and_token_limits(self):
        self.runtime.store.mutate(self.run['id'], lambda r: r.update(budget=dict(r['budget'], max_model_calls=1)))
        self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
        with self.assertRaises(SkillError) as err:
            self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
        self.assertEqual(err.exception.code, 'MODEL_CALL_BUDGET_EXCEEDED')

    def test_concurrent_reservation_has_one_winner(self):
        def reserve(_):
            try:
                self.runtime.store.reserve(self.run['id'], 'fixture', [], 100)
                return 'won'
            except SkillError:
                return 'denied'
        with ThreadPoolExecutor(max_workers=2) as pool:
            self.assertCountEqual(list(pool.map(reserve, range(2))), ['won', 'denied'])

    def test_invalid_usage_keeps_reservation(self):
        class Invalid:
            name = 'invalid'
            def complete(self, *args):
                return ModelReply({'summary': 'x'}, True, 10, 'x')
        with self.assertRaises(SkillError) as err:
            LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
        self.assertEqual(err.exception.code, 'MODEL_USAGE_INVALID')
        self.assertIsNone(self.runtime.store.calls(self.run['id'])[0]['charged'])

    def test_untrusted_extra_schema_fields_rejected_after_charging(self):
        class Invalid:
            name = 'invalid'
            def complete(self, *args):
                return ModelReply({'summary': 'x', 'shell': 'untrusted'}, 10, 10, 'x')
        with self.assertRaises(SkillError):
            LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
        self.assertEqual(self.runtime.store.calls(self.run['id'])[0]['charged'], 20)

    def test_context_overflow_rejected_before_provider(self):
        with self.assertRaises(SkillError):
            self.runtime.llm.generate(self.run['id'], 'design', {'data': 'x' * 30000}, lambda: False)
        self.assertEqual(self.model.count, 0)

    def test_dependency_policy_and_message_content_budget_preserve_source(self):
        captured=[]
        class Inspect:
            name='inspect-approved-contract'
            def complete(self,messages,max_tokens,cancelled):
                captured.append(messages)
                return ModelReply({'files':{'src/main/java/demo/App.java':'package demo; class App {}'}},80,80,self.name)
        # Escaping JSON for transport must not count its escape bytes a second time.
        source='String value = "quoted";\n'*500
        payload={'previous':{'src/main/java/demo/App.java':source},
                 'runtime_contract':{'dependencies':[{'coordinate':'org.apache.pdfbox:pdfbox-app','version':'3.0.8'}]}}
        before=self.runtime.store.get(self.run['id'])['budget']
        LLMGateway(self.runtime.store,[Inspect()]).generate(self.run['id'],'repair',payload,lambda:False)
        self.assertIn('explicitly pinned',captured[0][0]['content'])
        self.assertEqual(json.loads(captured[0][1]['content'])['data']['previous']['src/main/java/demo/App.java'],source)
        self.assertEqual(self.runtime.store.get(self.run['id'])['budget'],before)


class ProviderTests(unittest.TestCase):
    def provider(self, handler):
        return ChatCompletionProvider(name='fixture', endpoint='https://model.example/v1/chat/completions',
                                      model='approved-model', token='synthetic-secret', transport=httpx.MockTransport(handler))

    def test_success_preserves_fixed_route_and_usage(self):
        def respond(request):
            self.assertEqual(request.url.host, 'model.example')
            self.assertEqual(json.loads(request.content)['model'], 'approved-model')
            return httpx.Response(200, json={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"summary":"ok"}'}}],
                                             'usage': {'prompt_tokens': 4, 'completion_tokens': 5}})
        reply = self.provider(respond).complete([], 100, lambda: False)
        self.assertEqual(reply.content, {'summary': 'ok'})
        self.assertEqual(reply.output_tokens, 5)

    def test_status_classification_no_redirect_or_5xx_retry(self):
        for status, rejected, retry in [(429, True, True), (402, True, True), (401, True, False),
                                        (500, False, False), (302, False, False)]:
            with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
                self.provider(lambda r: httpx.Response(status, text='sensitive raw response')).complete([], 100, lambda: False)
            self.assertEqual((err.exception.rejected, err.exception.retryable), (rejected, retry))
            self.assertNotIn('sensitive', str(err.exception))

    def test_invalid_endpoint_and_unparsed_response(self):
        with self.assertRaises(ValueError):
            ChatCompletionProvider(name='x', endpoint='http://unsafe.example', model='x', token='x')
        for body in [{}, {'choices': [{'finish_reason': 'length', 'message': {'content': '{}'}}]}]:
            with self.assertRaises(ProviderFailure):
                self.provider(lambda r: httpx.Response(200, json=body)).complete([], 100, lambda: False)

    def test_authentication_failure_never_becomes_quota_fallback(self):
        for status, retry in [(401, False), (403, True)]:
            with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
                self.provider(lambda r: httpx.Response(status, json={'error': {'code': 'AllocationQuota'}})).complete([], 100, lambda: False)
            self.assertEqual(err.exception.retryable, retry)
            self.assertEqual(err.exception.details['http_status'], status)
            self.assertNotIn('AllocationQuota', str(err.exception))

    def test_cancellation_during_response_becomes_unknown(self):
        with self.assertRaises(ProviderFailure) as err:
            self.provider(lambda r: httpx.Response(200, content=b'{}')).complete([], 100, lambda: True)
        self.assertFalse(err.exception.rejected)