VibeCoding / src /tests /runtime /test_llm.py
dellweb3's picture
sol6.1 ERP稳定版本
115b94d verified
Raw History Blame Contribute Delete
8.31 kB
import json
import unittest
from concurrent.futures import ThreadPoolExecutor
import httpx
from src.adapters.llm.chat_completion import ChatCompletionProvider
from src.modules.llm_gateway.service import LLMGateway, ModelReply, ProviderFailure
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup, workspace, USER, ModelFixture
class LLMTests(unittest.TestCase):
def setUp(self):
self.runtime, request, self.model, _ = setup(self.enterContext(workspace()))
self.run = self.runtime.create(USER, request, 'model-test-key')
self.runtime.store.mutate(self.run['id'], lambda r: r.update(status='running', claim='test'))
def test_usage_settled_and_prompt_not_in_ledger(self):
result = self.runtime.llm.generate(self.run['id'], 'design', {'requirement': 'private requirement'}, lambda: False)
self.assertIn('summary', result)
calls = self.runtime.store.calls(self.run['id'])
self.assertEqual(calls[0]['charged'], 160)
self.assertNotIn('private requirement', json.dumps(calls))
def test_quota_fallback_counts_both_attempts(self):
first = ModelFixture()
first.name = 'quota'
def quota():
raise ProviderFailure('MODEL_QUOTA_REJECTED', rejected=True, retryable=True)
first.hook = quota
gateway = LLMGateway(self.runtime.store, [first, self.model])
gateway.generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual([c['state'] for c in self.runtime.store.calls(self.run['id'])], ['rejected', 'settled'])
def test_unknown_does_not_fallback(self):
first = ModelFixture()
def unknown():
raise ProviderFailure()
first.hook = unknown
with self.assertRaises(SkillError):
LLMGateway(self.runtime.store, [first, self.model]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(self.model.count, 0)
def test_model_count_and_token_limits(self):
self.runtime.store.mutate(self.run['id'], lambda r: r.update(budget=dict(r['budget'], max_model_calls=1)))
self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
with self.assertRaises(SkillError) as err:
self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(err.exception.code, 'MODEL_CALL_BUDGET_EXCEEDED')
def test_concurrent_reservation_has_one_winner(self):
def reserve(_):
try:
self.runtime.store.reserve(self.run['id'], 'fixture', [], 100)
return 'won'
except SkillError:
return 'denied'
with ThreadPoolExecutor(max_workers=2) as pool:
self.assertCountEqual(list(pool.map(reserve, range(2))), ['won', 'denied'])
def test_invalid_usage_keeps_reservation(self):
class Invalid:
name = 'invalid'
def complete(self, *args):
return ModelReply({'summary': 'x'}, True, 10, 'x')
with self.assertRaises(SkillError) as err:
LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(err.exception.code, 'MODEL_USAGE_INVALID')
self.assertIsNone(self.runtime.store.calls(self.run['id'])[0]['charged'])
def test_untrusted_extra_schema_fields_rejected_after_charging(self):
class Invalid:
name = 'invalid'
def complete(self, *args):
return ModelReply({'summary': 'x', 'shell': 'untrusted'}, 10, 10, 'x')
with self.assertRaises(SkillError):
LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False)
self.assertEqual(self.runtime.store.calls(self.run['id'])[0]['charged'], 20)
def test_context_overflow_rejected_before_provider(self):
with self.assertRaises(SkillError):
self.runtime.llm.generate(self.run['id'], 'design', {'data': 'x' * 30000}, lambda: False)
self.assertEqual(self.model.count, 0)
def test_dependency_policy_and_message_content_budget_preserve_source(self):
captured=[]
class Inspect:
name='inspect-approved-contract'
def complete(self,messages,max_tokens,cancelled):
captured.append(messages)
return ModelReply({'files':{'src/main/java/demo/App.java':'package demo; class App {}'}},80,80,self.name)
# Escaping JSON for transport must not count its escape bytes a second time.
source='String value = "quoted";\n'*500
payload={'previous':{'src/main/java/demo/App.java':source},
'runtime_contract':{'dependencies':[{'coordinate':'org.apache.pdfbox:pdfbox-app','version':'3.0.8'}]}}
before=self.runtime.store.get(self.run['id'])['budget']
LLMGateway(self.runtime.store,[Inspect()]).generate(self.run['id'],'repair',payload,lambda:False)
self.assertIn('explicitly pinned',captured[0][0]['content'])
self.assertEqual(json.loads(captured[0][1]['content'])['data']['previous']['src/main/java/demo/App.java'],source)
self.assertEqual(self.runtime.store.get(self.run['id'])['budget'],before)
class ProviderTests(unittest.TestCase):
def provider(self, handler):
return ChatCompletionProvider(name='fixture', endpoint='https://model.example/v1/chat/completions',
model='approved-model', token='synthetic-secret', transport=httpx.MockTransport(handler))
def test_success_preserves_fixed_route_and_usage(self):
def respond(request):
self.assertEqual(request.url.host, 'model.example')
self.assertEqual(json.loads(request.content)['model'], 'approved-model')
return httpx.Response(200, json={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"summary":"ok"}'}}],
'usage': {'prompt_tokens': 4, 'completion_tokens': 5}})
reply = self.provider(respond).complete([], 100, lambda: False)
self.assertEqual(reply.content, {'summary': 'ok'})
self.assertEqual(reply.output_tokens, 5)
def test_status_classification_no_redirect_or_5xx_retry(self):
for status, rejected, retry in [(429, True, True), (402, True, True), (401, True, False),
(500, False, False), (302, False, False)]:
with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(status, text='sensitive raw response')).complete([], 100, lambda: False)
self.assertEqual((err.exception.rejected, err.exception.retryable), (rejected, retry))
self.assertNotIn('sensitive', str(err.exception))
def test_invalid_endpoint_and_unparsed_response(self):
with self.assertRaises(ValueError):
ChatCompletionProvider(name='x', endpoint='http://unsafe.example', model='x', token='x')
for body in [{}, {'choices': [{'finish_reason': 'length', 'message': {'content': '{}'}}]}]:
with self.assertRaises(ProviderFailure):
self.provider(lambda r: httpx.Response(200, json=body)).complete([], 100, lambda: False)
def test_authentication_failure_never_becomes_quota_fallback(self):
for status, retry in [(401, False), (403, True)]:
with self.subTest(status=status), self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(status, json={'error': {'code': 'AllocationQuota'}})).complete([], 100, lambda: False)
self.assertEqual(err.exception.retryable, retry)
self.assertEqual(err.exception.details['http_status'], status)
self.assertNotIn('AllocationQuota', str(err.exception))
def test_cancellation_during_response_becomes_unknown(self):
with self.assertRaises(ProviderFailure) as err:
self.provider(lambda r: httpx.Response(200, content=b'{}')).complete([], 100, lambda: True)
self.assertFalse(err.exception.rejected)