Spaces:
Sleeping
Sleeping
File size: 8,305 Bytes
115b94d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | import json
import unittest
from concurrent.futures import ThreadPoolExecutor
import httpx
from src.adapters.llm.chat_completion import ChatCompletionProvider
from src.modules.llm_gateway.service import LLMGateway, ModelReply, ProviderFailure
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup, workspace, USER, ModelFixture
class LLMTests(unittest.TestCase):
def setUp(self):
self.runtime, request, self.model, _ = setup(self.enterContext(workspace()))
self.run = self.runtime.create(USER, request, 'model-test-key')
self.runtime.store.mutate(self.run['id'], lambda r: r.update(status='running', claim='test'))
def test_usage_settled_and_prompt_not_in_ledger(self):
result = self.runtime.llm.generate(self.run['id'], 'design', {'requirement': 'private requirement'}, lambda: False)
self.assertIn('summary', result)
calls = self.runtime.store.calls(self.run['id'])
self.assertEqual(calls[0]['charged'], 160)
self.assertNotIn('private requirement', json.dumps(calls))
def test_quota_fallback_counts_both_attempts(self):
first = ModelFixture()
first.name = 'quota'
def quota():
raise ProviderFailure('MODEL_QUOTA_REJECTED', rejected=True, retryable=True)
first.hook = quota
gateway = LLMGateway(self.runtime.store, [first, self.model])
gateway.generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual([c['state'] for c in self.runtime.store.calls(self.run['id'])], ['rejected', 'settled'])
def test_unknown_does_not_fallback(self):
first = ModelFixture()
def unknown():
raise ProviderFailure()
first.hook = unknown
with self.assertRaises(SkillError):
LLMGateway(self.runtime.store, [first, self.model]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(self.model.count, 0)
def test_model_count_and_token_limits(self):
self.runtime.store.mutate(self.run['id'], lambda r: r.update(budget=dict(r['budget'], max_model_calls=1)))
self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
with self.assertRaises(SkillError) as err:
self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(err.exception.code, 'MODEL_CALL_BUDGET_EXCEEDED')
def test_concurrent_reservation_has_one_winner(self):
def reserve(_):
try:
self.runtime.store.reserve(self.run['id'], 'fixture', [], 100)
return 'won'
except SkillError:
return 'denied'
with ThreadPoolExecutor(max_workers=2) as pool:
self.assertCountEqual(list(pool.map(reserve, range(2))), ['won', 'denied'])
def test_invalid_usage_keeps_reservation(self):
class Invalid:
name = 'invalid'
def complete(self, *args):
return ModelReply({'summary': 'x'}, True, 10, 'x')
with self.assertRaises(SkillError) as err:
LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(err.exception.code, 'MODEL_USAGE_INVALID')
self.assertIsNone(self.runtime.store.calls(self.run['id'])[0]['charged'])
def test_untrusted_extra_schema_fields_rejected_after_charging(self):
class Invalid:
name = 'invalid'
def complete(self, *args):
return ModelReply({'summary': 'x', 'shell': 'untrusted'}, 10, 10, 'x')
with self.assertRaises(SkillError):
LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(self.runtime.store.calls(self.run['id'])[0]['charged'], 20)
def test_context_overflow_rejected_before_provider(self):
with self.assertRaises(SkillError):
self.runtime.llm.generate(self.run['id'], 'design', {'data': 'x' * 30000}, lambda: False)
self.assertEqual(self.model.count, 0)
def test_dependency_policy_and_message_content_budget_preserve_source(self):
captured=[]
class Inspect:
name='inspect-approved-contract'
def complete(self,messages,max_tokens,cancelled):
captured.append(messages)
return ModelReply({'files':{'src/main/java/demo/App.java':'package demo; class App {}'}},80,80,self.name)
# Escaping JSON for transport must not count its escape bytes a second time.
source='String value = "quoted";\n'*500
payload={'previous':{'src/main/java/demo/App.java':source},
'runtime_contract':{'dependencies':[{'coordinate':'org.apache.pdfbox:pdfbox-app','version':'3.0.8'}]}}
before=self.runtime.store.get(self.run['id'])['budget']
LLMGateway(self.runtime.store,[Inspect()]).generate(self.run['id'],'repair',payload,lambda:False)
self.assertIn('explicitly pinned',captured[0][0]['content'])
self.assertEqual(json.loads(captured[0][1]['content'])['data']['previous']['src/main/java/demo/App.java'],source)
self.assertEqual(self.runtime.store.get(self.run['id'])['budget'],before)
class ProviderTests(unittest.TestCase):
def provider(self, handler):
return ChatCompletionProvider(name='fixture', endpoint='https://model.example/v1/chat/completions',
model='approved-model', token='synthetic-secret', transport=httpx.MockTransport(handler))
def test_success_preserves_fixed_route_and_usage(self):
def respond(request):
self.assertEqual(request.url.host, 'model.example')
self.assertEqual(json.loads(request.content)['model'], 'approved-model')
return httpx.Response(200, json={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"summary":"ok"}'}}],
'usage': {'prompt_tokens': 4, 'completion_tokens': 5}})
reply = self.provider(respond).complete([], 100, lambda: False)
self.assertEqual(reply.content, {'summary': 'ok'})
self.assertEqual(reply.output_tokens, 5)
def test_status_classification_no_redirect_or_5xx_retry(self):
for status, rejected, retry in [(429, True, True), (402, True, True), (401, True, False),
(500, False, False), (302, False, False)]:
with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(status, text='sensitive raw response')).complete([], 100, lambda: False)
self.assertEqual((err.exception.rejected, err.exception.retryable), (rejected, retry))
self.assertNotIn('sensitive', str(err.exception))
def test_invalid_endpoint_and_unparsed_response(self):
with self.assertRaises(ValueError):
ChatCompletionProvider(name='x', endpoint='http://unsafe.example', model='x', token='x')
for body in [{}, {'choices': [{'finish_reason': 'length', 'message': {'content': '{}'}}]}]:
with self.assertRaises(ProviderFailure):
self.provider(lambda r: httpx.Response(200, json=body)).complete([], 100, lambda: False)
def test_authentication_failure_never_becomes_quota_fallback(self):
for status, retry in [(401, False), (403, True)]:
with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(status, json={'error': {'code': 'AllocationQuota'}})).complete([], 100, lambda: False)
self.assertEqual(err.exception.retryable, retry)
self.assertEqual(err.exception.details['http_status'], status)
self.assertNotIn('AllocationQuota', str(err.exception))
def test_cancellation_during_response_becomes_unknown(self):
with self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(200, content=b'{}')).complete([], 100, lambda: True)
self.assertFalse(err.exception.rejected)
|