Spaces:
Sleeping
Sleeping
Download src/tests/runtime/test_llm.py from dellweb3/VibeCoding: direct link, hf CLI and curl.
- Browser
- Download file 8.31 kB
-
https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/tests/runtime/test_llm.py
- Command line
-
hf download hf://spaces/dellweb3/VibeCoding/src/tests/runtime/test_llm.py
-
curl -L -o test_llm.py https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/tests/runtime/test_llm.py
8.31 kB
| import json | |
| import unittest | |
| from concurrent.futures import ThreadPoolExecutor | |
| import httpx | |
| from src.adapters.llm.chat_completion import ChatCompletionProvider | |
| from src.modules.llm_gateway.service import LLMGateway, ModelReply, ProviderFailure | |
| from src.modules.skill.domain.models import SkillError | |
| from src.tests.runtime.fixtures import setup, workspace, USER, ModelFixture | |
| class LLMTests(unittest.TestCase): | |
| def setUp(self): | |
| self.runtime, request, self.model, _ = setup(self.enterContext(workspace())) | |
| self.run = self.runtime.create(USER, request, 'model-test-key') | |
| self.runtime.store.mutate(self.run['id'], lambda r: r.update(status='running', claim='test')) | |
| def test_usage_settled_and_prompt_not_in_ledger(self): | |
| result = self.runtime.llm.generate(self.run['id'], 'design', {'requirement': 'private requirement'}, lambda: False) | |
| self.assertIn('summary', result) | |
| calls = self.runtime.store.calls(self.run['id']) | |
| self.assertEqual(calls[0]['charged'], 160) | |
| self.assertNotIn('private requirement', json.dumps(calls)) | |
| def test_quota_fallback_counts_both_attempts(self): | |
| first = ModelFixture() | |
| first.name = 'quota' | |
| def quota(): | |
| raise ProviderFailure('MODEL_QUOTA_REJECTED', rejected=True, retryable=True) | |
| first.hook = quota | |
| gateway = LLMGateway(self.runtime.store, [first, self.model]) | |
| gateway.generate(self.run['id'], 'design', {}, lambda: False) | |
| self.assertEqual([c['state'] for c in self.runtime.store.calls(self.run['id'])], ['rejected', 'settled']) | |
| def test_unknown_does_not_fallback(self): | |
| first = ModelFixture() | |
| def unknown(): | |
| raise ProviderFailure() | |
| first.hook = unknown | |
| with self.assertRaises(SkillError): | |
| LLMGateway(self.runtime.store, [first, self.model]).generate(self.run['id'], 'design', {}, lambda: False) | |
| self.assertEqual(self.model.count, 0) | |
| def test_model_count_and_token_limits(self): | |
| self.runtime.store.mutate(self.run['id'], lambda r: r.update(budget=dict(r['budget'], max_model_calls=1))) | |
| self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False) | |
| with self.assertRaises(SkillError) as err: | |
| self.runtime.llm.generate(self.run['id'], 'design', {}, lambda: False) | |
| self.assertEqual(err.exception.code, 'MODEL_CALL_BUDGET_EXCEEDED') | |
| def test_concurrent_reservation_has_one_winner(self): | |
| def reserve(_): | |
| try: | |
| self.runtime.store.reserve(self.run['id'], 'fixture', [], 100) | |
| return 'won' | |
| except SkillError: | |
| return 'denied' | |
| with ThreadPoolExecutor(max_workers=2) as pool: | |
| self.assertCountEqual(list(pool.map(reserve, range(2))), ['won', 'denied']) | |
| def test_invalid_usage_keeps_reservation(self): | |
| class Invalid: | |
| name = 'invalid' | |
| def complete(self, *args): | |
| return ModelReply({'summary': 'x'}, True, 10, 'x') | |
| with self.assertRaises(SkillError) as err: | |
| LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False) | |
| self.assertEqual(err.exception.code, 'MODEL_USAGE_INVALID') | |
| self.assertIsNone(self.runtime.store.calls(self.run['id'])[0]['charged']) | |
| def test_untrusted_extra_schema_fields_rejected_after_charging(self): | |
| class Invalid: | |
| name = 'invalid' | |
| def complete(self, *args): | |
| return ModelReply({'summary': 'x', 'shell': 'untrusted'}, 10, 10, 'x') | |
| with self.assertRaises(SkillError): | |
| LLMGateway(self.runtime.store, [Invalid()]).generate(self.run['id'], 'design', {}, lambda: False) | |
| self.assertEqual(self.runtime.store.calls(self.run['id'])[0]['charged'], 20) | |
| def test_context_overflow_rejected_before_provider(self): | |
| with self.assertRaises(SkillError): | |
| self.runtime.llm.generate(self.run['id'], 'design', {'data': 'x' * 30000}, lambda: False) | |
| self.assertEqual(self.model.count, 0) | |
| def test_dependency_policy_and_message_content_budget_preserve_source(self): | |
| captured=[] | |
| class Inspect: | |
| name='inspect-approved-contract' | |
| def complete(self,messages,max_tokens,cancelled): | |
| captured.append(messages) | |
| return ModelReply({'files':{'src/main/java/demo/App.java':'package demo; class App {}'}},80,80,self.name) | |
| # Escaping JSON for transport must not count its escape bytes a second time. | |
| source='String value = "quoted";\n'*500 | |
| payload={'previous':{'src/main/java/demo/App.java':source}, | |
| 'runtime_contract':{'dependencies':[{'coordinate':'org.apache.pdfbox:pdfbox-app','version':'3.0.8'}]}} | |
| before=self.runtime.store.get(self.run['id'])['budget'] | |
| LLMGateway(self.runtime.store,[Inspect()]).generate(self.run['id'],'repair',payload,lambda:False) | |
| self.assertIn('explicitly pinned',captured[0][0]['content']) | |
| self.assertEqual(json.loads(captured[0][1]['content'])['data']['previous']['src/main/java/demo/App.java'],source) | |
| self.assertEqual(self.runtime.store.get(self.run['id'])['budget'],before) | |
| class ProviderTests(unittest.TestCase): | |
| def provider(self, handler): | |
| return ChatCompletionProvider(name='fixture', endpoint='https://model.example/v1/chat/completions', | |
| model='approved-model', token='synthetic-secret', transport=httpx.MockTransport(handler)) | |
| def test_success_preserves_fixed_route_and_usage(self): | |
| def respond(request): | |
| self.assertEqual(request.url.host, 'model.example') | |
| self.assertEqual(json.loads(request.content)['model'], 'approved-model') | |
| return httpx.Response(200, json={'choices': [{'finish_reason': 'stop', 'message': {'content': '{"summary":"ok"}'}}], | |
| 'usage': {'prompt_tokens': 4, 'completion_tokens': 5}}) | |
| reply = self.provider(respond).complete([], 100, lambda: False) | |
| self.assertEqual(reply.content, {'summary': 'ok'}) | |
| self.assertEqual(reply.output_tokens, 5) | |
| def test_status_classification_no_redirect_or_5xx_retry(self): | |
| for status, rejected, retry in [(429, True, True), (402, True, True), (401, True, False), | |
| (500, False, False), (302, False, False)]: | |
| with self.subTest(status=status), self.assertRaises(ProviderFailure) as err: | |
| self.provider(lambda r: httpx.Response(status, text='sensitive raw response')).complete([], 100, lambda: False) | |
| self.assertEqual((err.exception.rejected, err.exception.retryable), (rejected, retry)) | |
| self.assertNotIn('sensitive', str(err.exception)) | |
| def test_invalid_endpoint_and_unparsed_response(self): | |
| with self.assertRaises(ValueError): | |
| ChatCompletionProvider(name='x', endpoint='http://unsafe.example', model='x', token='x') | |
| for body in [{}, {'choices': [{'finish_reason': 'length', 'message': {'content': '{}'}}]}]: | |
| with self.assertRaises(ProviderFailure): | |
| self.provider(lambda r: httpx.Response(200, json=body)).complete([], 100, lambda: False) | |
| def test_authentication_failure_never_becomes_quota_fallback(self): | |
| for status, retry in [(401, False), (403, True)]: | |
| with self.subTest(status=status), self.assertRaises(ProviderFailure) as err: | |
| self.provider(lambda r: httpx.Response(status, json={'error': {'code': 'AllocationQuota'}})).complete([], 100, lambda: False) | |
| self.assertEqual(err.exception.retryable, retry) | |
| self.assertEqual(err.exception.details['http_status'], status) | |
| self.assertNotIn('AllocationQuota', str(err.exception)) | |
| def test_cancellation_during_response_becomes_unknown(self): | |
| with self.assertRaises(ProviderFailure) as err: | |
| self.provider(lambda r: httpx.Response(200, content=b'{}')).complete([], 100, lambda: True) | |
| self.assertFalse(err.exception.rejected) | |