import json import unittest from concurrent.futures import ThreadPoolExecutor import httpx from src.adapters.llm.chat_completion import ChatCompletionProvider from src.modules.llm_gateway.service import LLMGateway, ModelReply, ProviderFailure from src.modules.skill.domain.models import SkillError from src.tests.runtime.fixtures import setup, workspace, USER, ModelFixture class LLMTests(unittest.TestCase): def setUp(self): self.runtime, request, self.model, _ = setup(self.enterContext(workspace())) self.run = self.runtime.create(USER, request, 'model-test-key') self.runtime.store.mutate(self.run['id'], lambda r: r.update(status='running', claim='test')) def test_usage_settled_and_prompt_not_in_ledger(self): result = self.runtime.llm.generate(self.run['id'], 'design', {'requirement': 'private requirement'}, lambda: False) self.assertIn('summary', result) calls = self.runtime.store.calls(self.run['id']) self.assertEqual(calls[0]['charged'], 160) self.assertNotIn('private requirement', json.dumps(calls)) def test_quota_fallback_counts_both_attempts(self): first = ModelFixture() first.name = 'quota' def quota(): raise ProviderFailure('MODEL_QUOTA_REJECTED', rejected=True, retryable=True) first.hook = quota gateway = LLMGateway(self.runtime.store, [first, self.model]) gateway.generate(self.run['id'], 'design', {}, lambda: False) self.assertEqual([c['state'] for c in self.runtime.store.calls(self.run['id'])], ['rejected', 'settled']) def test_unknown_does_not_fallback(self): first = ModelFixture() def unknown(): raise ProviderFailure() first.hook = unknown with self.assertRaises(SkillError): LLMGateway(self.runtime.store, [first, self.model]).generate(self.run['id'], 'design', {}, lambda: False) self.assertEqual(self.model.count, 0) def test_model_count_and_token_limits(self): self.runtime.store.mutate(self.run['id'], lambda r: r.update(budget=dict(r['budget'], max_model_calls=1))) self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False) with self.assertRaises(SkillError) as err: self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False) self.assertEqual(err.exception.code, 'MODEL_CALL_BUDGET_EXCEEDED') def test_concurrent_reservation_has_one_winner(self): def reserve(_): try: self.runtime.store.reserve(self.run['id'], 'fixture', [], 100) return 'won' except SkillError: return 'denied' with ThreadPoolExecutor(max_workers=2) as pool: self.assertCountEqual(list(pool.map(reserve, range(2))), ['won', 'denied']) def test_invalid_usage_keeps_reservation(self): class Invalid: name = 'invalid' def complete(self, *args): return ModelReply({'summary': 'x'}, True, 10, 'x') with self.assertRaises(SkillError) as err: LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False) self.assertEqual(err.exception.code, 'MODEL_USAGE_INVALID') self.assertIsNone(self.runtime.store.calls(self.run['id'])[0]['charged']) def test_untrusted_extra_schema_fields_rejected_after_charging(self): class Invalid: name = 'invalid' def complete(self, *args): return ModelReply({'summary': 'x', 'shell': 'untrusted'}, 10, 10, 'x') with self.assertRaises(SkillError): LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False) self.assertEqual(self.runtime.store.calls(self.run['id'])[0]['charged'], 20) def test_context_overflow_rejected_before_provider(self): with self.assertRaises(SkillError): self.runtime.llm.generate(self.run['id'], 'design', {'data': 'x' * 30000}, lambda: False) self.assertEqual(self.model.count, 0) def test_dependency_policy_and_message_content_budget_preserve_source(self): captured=[] class Inspect: name='inspect-approved-contract' def complete(self,messages,max_tokens,cancelled): captured.append(messages) return ModelReply({'files':{'src/main/java/demo/App.java':'package demo; class App {}'}},80,80,self.name) # Escaping JSON for transport must not count its escape bytes a second time. source='String value = "quoted";\n'*500 payload={'previous':{'src/main/java/demo/App.java':source}, 'runtime_contract':{'dependencies':[{'coordinate':'org.apache.pdfbox:pdfbox-app','version':'3.0.8'}]}} before=self.runtime.store.get(self.run['id'])['budget'] LLMGateway(self.runtime.store,[Inspect()]).generate(self.run['id'],'repair',payload,lambda:False) self.assertIn('explicitly pinned',captured[0][0]['content']) self.assertEqual(json.loads(captured[0][1]['content'])['data']['previous']['src/main/java/demo/App.java'],source) self.assertEqual(self.runtime.store.get(self.run['id'])['budget'],before) class ProviderTests(unittest.TestCase): def provider(self, handler): return ChatCompletionProvider(name='fixture', endpoint='https://model.example/v1/chat/completions', model='approved-model', token='synthetic-secret', transport=httpx.MockTransport(handler)) def test_success_preserves_fixed_route_and_usage(self): def respond(request): self.assertEqual(request.url.host, 'model.example') self.assertEqual(json.loads(request.content)['model'], 'approved-model') return httpx.Response(200, json={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"summary":"ok"}'}}], 'usage': {'prompt_tokens': 4, 'completion_tokens': 5}}) reply = self.provider(respond).complete([], 100, lambda: False) self.assertEqual(reply.content, {'summary': 'ok'}) self.assertEqual(reply.output_tokens, 5) def test_status_classification_no_redirect_or_5xx_retry(self): for status, rejected, retry in [(429, True, True), (402, True, True), (401, True, False), (500, False, False), (302, False, False)]: with self.subTest(status=status), self.assertRaises(ProviderFailure) as err: self.provider(lambda r: httpx.Response(status, text='sensitive raw response')).complete([], 100, lambda: False) self.assertEqual((err.exception.rejected, err.exception.retryable), (rejected, retry)) self.assertNotIn('sensitive', str(err.exception)) def test_invalid_endpoint_and_unparsed_response(self): with self.assertRaises(ValueError): ChatCompletionProvider(name='x', endpoint='http://unsafe.example', model='x', token='x') for body in [{}, {'choices': [{'finish_reason': 'length', 'message': {'content': '{}'}}]}]: with self.assertRaises(ProviderFailure): self.provider(lambda r: httpx.Response(200, json=body)).complete([], 100, lambda: False) def test_authentication_failure_never_becomes_quota_fallback(self): for status, retry in [(401, False), (403, True)]: with self.subTest(status=status), self.assertRaises(ProviderFailure) as err: self.provider(lambda r: httpx.Response(status, json={'error': {'code': 'AllocationQuota'}})).complete([], 100, lambda: False) self.assertEqual(err.exception.retryable, retry) self.assertEqual(err.exception.details['http_status'], status) self.assertNotIn('AllocationQuota', str(err.exception)) def test_cancellation_during_response_becomes_unknown(self): with self.assertRaises(ProviderFailure) as err: self.provider(lambda r: httpx.Response(200, content=b'{}')).complete([], 100, lambda: True) self.assertFalse(err.exception.rejected)