Spaces:
Sleeping
Sleeping
Download src/tests/runtime/test_bailian_gateway.py from dellweb3/VibeCoding: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/tests/runtime/test_bailian_gateway.py
- Command line
-
hf download hf://spaces/dellweb3/VibeCoding/src/tests/runtime/test_bailian_gateway.py
-
curl -L -o test_bailian_gateway.py https://huggingface.co/spaces/dellweb3/VibeCoding/resolve/main/src/tests/runtime/test_bailian_gateway.py
21.5 kB
| """Protocol/fault tests with explicit synthetic providers; live Java evidence is separate.""" | |
| import json | |
| import sqlite3 | |
| import unittest | |
| from pathlib import Path | |
| import httpx | |
| from pydantic import ValidationError | |
| from src.adapters.llm.chat_completion import ChatCompletionProvider | |
| from src.adapters.persistence.runtime_sqlite import RuntimeStore | |
| from src.modules.llm_gateway.routing import VERSION,plan | |
| from src.modules.llm_gateway.service import LLMGateway,ModelReply,ProviderFailure | |
| from src.modules.skill.domain.models import Budget,SkillError | |
| from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE,AUTHOR | |
| class BailianGatewayTests(unittest.TestCase): | |
| def setUp(self): | |
| self.folder=Path(self.enterContext(workspace())) | |
| self.runtime,request,_,_=setup(self.folder) | |
| self.run=self.runtime.create(USER,request,'bailian-fault-test') | |
| self.runtime.store.mutate(self.run['id'],lambda r:r.update(status='running',claim='test')) | |
| def provider(self,handler,**options): | |
| return ChatCompletionProvider(name='synthetic-bailian',endpoint='https://model.example/v1/chat/completions', | |
| model='synthetic-approved-model',token='synthetic-secret',transport=httpx.MockTransport(handler),**options) | |
| def response(self,content,reason='stop',output=20): | |
| return httpx.Response(200,json={'id':'test-request-123','choices':[{'finish_reason':reason,'message':{'content':content}}], | |
| 'usage':{'prompt_tokens':10,'completion_tokens':output}}) | |
| def generate(self,providers,kind='code',cancelled=lambda:False,payload=None): | |
| return LLMGateway(self.runtime.store,providers).generate(self.run['id'],kind,payload or {},cancelled) | |
| def calls(self):return self.runtime.store.calls(self.run['id']) | |
| def test_normal_generation_is_one_call_with_larger_code_budget(self): | |
| seen=[] | |
| def respond(request): | |
| seen.append(json.loads(request.content)) | |
| return self.response(json.dumps({'files':SOURCE})) | |
| self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) | |
| self.assertEqual(len(seen),1) | |
| self.assertEqual(seen[0]['max_tokens'],8192) | |
| self.assertEqual(self.calls()[0]['charged'],30) | |
| self.assertEqual(self.calls()[0]['metadata']['request_id'],'test-request-123') | |
| def test_business_missing_page_is_known_invalid_and_regenerated_once(self): | |
| self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) | |
| seen=[] | |
| def respond(request): | |
| seen.append(request) | |
| files={'src/main/java/demo/App.java':'class App {}'} | |
| if len(seen)>1:files['website/index.html']='<html><style></style><script></script></html>' | |
| return self.response(json.dumps({'files':files})) | |
| result=self.generate([self.provider(respond)]) | |
| self.assertEqual(set(result['files']),{'src/main/java/demo/App.java','website/index.html'}) | |
| calls=self.calls();self.assertEqual([c['state'] for c in calls],['settled','settled']) | |
| self.assertEqual(calls[0]['metadata']['error'],'MODEL_OUTPUT_INVALID') | |
| self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id']) | |
| def test_business_repair_can_correct_java_and_page_contract_together(self): | |
| self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) | |
| files={**SOURCE,'website/index.html':'<html><script>const rows=data.items;</script></html>'} | |
| result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':files})))],kind='repair') | |
| self.assertEqual(result['files'],files) | |
| self.assertEqual(len(self.calls()),1) | |
| self.assertEqual(self.calls()[0]['state'],'settled') | |
| def test_explicit_java_only_continuation_still_retains_its_fixed_scope(self): | |
| def update(run): | |
| run['snapshot']['framework']['profile_id']='java17-requirement-business' | |
| run['request']['recovery_diagnostics']={'observed':'known Java failure'} | |
| self.runtime.store.mutate(self.run['id'],update) | |
| result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':SOURCE})))]) | |
| self.assertEqual(result['files'],SOURCE) | |
| self.assertEqual(len(self.calls()),1) | |
| def test_business_embedded_page_is_rejected_and_regenerated(self): | |
| self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) | |
| seen=[] | |
| def respond(request): | |
| seen.append(request) | |
| java='class App { String page = "<html>"; }' if len(seen)==1 else 'class App {}' | |
| return self.response(json.dumps({'files':{'src/main/java/demo/App.java':java,'website/index.html':'<html></html>'}})) | |
| result=self.generate([self.provider(respond)]) | |
| self.assertEqual(result['files']['src/main/java/demo/App.java'],'class App {}') | |
| self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) | |
| self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_INVALID') | |
| def test_known_truncation_settles_usage_then_regenerates_complete_json(self): | |
| seen=[] | |
| def respond(request): | |
| seen.append(json.loads(request.content)) | |
| return (self.response('{"files":',reason='length',output=seen[-1]['max_tokens']) if len(seen)==1 | |
| else self.response(json.dumps({'files':SOURCE}))) | |
| self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) | |
| self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) | |
| self.assertEqual(self.calls()[0]['charged'],8202) | |
| self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_TRUNCATED') | |
| self.assertGreater(seen[1]['max_tokens'],seen[0]['max_tokens']) | |
| self.assertEqual(self.calls()[1]['metadata']['retry_of'],self.calls()[0]['id']) | |
| self.assertTrue(any(e['stage']=='model_recovery' for e in self.runtime.store.stages(self.run['id']))) | |
| def test_repeated_truncation_stops_and_remains_known(self): | |
| with self.assertRaises(SkillError) as err: | |
| self.generate([self.provider(lambda r:self.response('{',reason='length'))]) | |
| self.assertEqual(err.exception.code,'MODEL_OUTPUT_TRUNCATED') | |
| self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) | |
| def test_invalid_json_and_invalid_schema_recover_only_once(self): | |
| for value in ('not json',json.dumps({'shell':'untrusted'})): | |
| with self.subTest(value=value): | |
| # Distinct run keys avoid treating two independent tests as retries. | |
| self.setUp() | |
| seen=[] | |
| def respond(request): | |
| seen.append(request) | |
| return self.response(value if len(seen)==1 else json.dumps({'files':SOURCE})) | |
| self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) | |
| self.assertEqual(len(self.calls()),2) | |
| def test_unknown_transport_never_falls_back(self): | |
| seen=[] | |
| def timeout(request):raise httpx.ReadTimeout('synthetic-sensitive-body') | |
| backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE}))) | |
| with self.assertRaises(SkillError) as err:self.generate([self.provider(timeout),backup]) | |
| self.assertEqual(err.exception.code,'MODEL_RESULT_UNKNOWN') | |
| self.assertEqual(seen,[]) | |
| self.assertEqual(self.calls()[0]['state'],'unknown') | |
| self.assertIsNone(self.calls()[0]['charged']) | |
| self.assertEqual(self.calls()[0]['metadata']['failure_category'],'transport_timeout') | |
| diagnostics=self.runtime.store.diagnostics(self.run['id']) | |
| self.assertTrue(any(d['component']=='llm-gateway' and d['code']=='MODEL_RESULT_UNKNOWN' for d in diagnostics)) | |
| self.assertNotIn('synthetic-sensitive',json.dumps(diagnostics)) | |
| self.assertNotIn('synthetic-secret',json.dumps(diagnostics)) | |
| self.assertNotIn('synthetic-sensitive',json.dumps(self.calls())) | |
| def test_length_without_usage_is_unknown_not_retried(self): | |
| seen=[] | |
| def respond(request): | |
| seen.append(request) | |
| return httpx.Response(200,json={'choices':[{'finish_reason':'length','message':{'content':'{'}}]}) | |
| with self.assertRaises(SkillError):self.generate([self.provider(respond)]) | |
| self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'unknown') | |
| def test_quota_rejection_falls_back_but_authentication_does_not(self): | |
| for status,retry in [(429,True),(401,False)]: | |
| with self.subTest(status=status): | |
| self.setUp();seen=[] | |
| first=self.provider(lambda r:httpx.Response(status,text='synthetic raw error')) | |
| backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE}))) | |
| if retry:self.generate([first,backup]) | |
| else: | |
| with self.assertRaises(SkillError):self.generate([first,backup]) | |
| self.assertEqual(len(seen),int(retry));self.assertEqual(self.calls()[0]['charged'],0) | |
| def test_content_filter_and_tool_calls_are_known_but_not_replayed(self): | |
| for reason in ('content_filter','tool_calls'): | |
| with self.subTest(reason=reason): | |
| self.setUp() | |
| with self.assertRaises(SkillError) as err:self.generate([self.provider(lambda r:self.response('{}',reason=reason))]) | |
| self.assertEqual(err.exception.code,'MODEL_OUTPUT_UNSUPPORTED') | |
| self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['state'],'settled') | |
| def test_rejected_provider_code_is_retained_without_sensitive_message(self): | |
| provider=self.provider(lambda r:httpx.Response(403,json={'error':{ | |
| 'code':'AllocationQuota.FreeTierOnly','message':'quota exhausted; synthetic-secret account details'}})) | |
| with self.assertRaises(SkillError):self.generate([provider]) | |
| call=self.calls()[0] | |
| self.assertEqual(call['metadata']['provider_error_code'],'AllocationQuota.FreeTierOnly') | |
| self.assertEqual(call['state'],'rejected') | |
| self.assertEqual(call['charged'],0) | |
| self.assertNotIn('synthetic-secret',json.dumps(call)) | |
| self.assertTrue(call['metadata']['response_evidence']['response_received']) | |
| def test_recovery_obeys_total_call_limit(self): | |
| self.runtime.store.mutate(self.run['id'],lambda r:r['budget'].update(max_model_calls=1)) | |
| seen=[] | |
| def respond(r):seen.append(r);return self.response('{',reason='length') | |
| with self.assertRaises(SkillError) as err:self.generate([self.provider(respond)]) | |
| self.assertEqual(err.exception.code,'MODEL_CALL_BUDGET_EXCEEDED') | |
| self.assertEqual(len(seen),1) | |
| def test_stage_with_no_room_to_expand_does_not_repeat_a_truncated_response(self): | |
| seen=[] | |
| def respond(request): | |
| value=json.loads(request.content);seen.append(value) | |
| return self.response('{',reason='length',output=value['max_tokens']) | |
| gateway=LLMGateway(self.runtime.store,[self.provider(respond)]) | |
| with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,2000) | |
| self.assertEqual(err.exception.code,'MODEL_OUTPUT_LIMIT') | |
| self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled') | |
| def test_project_revocation_is_rechecked_before_recovery_dispatch(self): | |
| seen=[];checks=[] | |
| def guard(): | |
| checks.append(True) | |
| if len(checks)>1:raise SkillError('PROJECT_NOT_FOUND','revoked',404) | |
| def respond(request):seen.append(request);return self.response('{',reason='length') | |
| gateway=LLMGateway(self.runtime.store,[self.provider(respond)]) | |
| with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,dispatch_guard=guard) | |
| self.assertEqual(err.exception.code,'PROJECT_NOT_FOUND') | |
| self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled') | |
| def test_cancellation_after_response_settles_and_never_retries(self): | |
| state={'cancelled':False} | |
| class Model: | |
| name='synthetic-cancelled' | |
| def complete(self,*args): | |
| state['cancelled']=True | |
| raise ProviderFailure('MODEL_OUTPUT_TRUNCATED',input_tokens=10,output_tokens=10) | |
| with self.assertRaises(SkillError) as err:self.generate([Model()],cancelled=lambda:state['cancelled']) | |
| self.assertEqual(err.exception.code,'RUN_CANCELLED') | |
| self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['charged'],20) | |
| def test_coder_profile_does_not_send_unsupported_json_or_thinking_parameters(self): | |
| def respond(request): | |
| value=json.loads(request.content) | |
| self.assertNotIn('response_format',value);self.assertNotIn('enable_thinking',value) | |
| return self.response(json.dumps({'files':SOURCE})) | |
| self.generate([self.provider(respond,json_mode=False)]) | |
| self.assertNotIn('synthetic-secret',json.dumps(self.calls())) | |
| self.assertNotIn('synthetic-secret',json.dumps(LLMGateway(self.runtime.store,[self.provider(respond)]).routes())) | |
| def test_known_invalid_review_recovers_once_and_keeps_safe_validation_reason(self): | |
| seen=[] | |
| def respond(request): | |
| seen.append(json.loads(request.content)) | |
| body={'summary':'review','findings':[{'secret':'sensitive-untrusted-value'}] if len(seen)==1 else [],'files':SOURCE} | |
| return self.response(json.dumps(body)) | |
| result=self.generate([self.provider(respond)],kind='code_review',payload={'patch':{'files':SOURCE}}) | |
| self.assertEqual(result['files'],SOURCE) | |
| calls=self.calls();self.assertEqual(len(calls),2) | |
| self.assertEqual([c['state'] for c in calls],['settled','settled']) | |
| self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id']) | |
| self.assertEqual(calls[0]['metadata']['validation_errors'],[{'location':['findings',0],'type':'string_type'}]) | |
| self.assertNotIn('sensitive-untrusted-value',json.dumps(calls)) | |
| def test_repeated_invalid_review_stops_within_shared_call_budget(self): | |
| with self.assertRaises(SkillError) as error: | |
| self.generate([self.provider(lambda r:self.response(json.dumps({'files':SOURCE})))],kind='code_review',payload={'patch':{'files':SOURCE}}) | |
| self.assertEqual(error.exception.code,'MODEL_OUTPUT_INVALID') | |
| self.assertEqual(len(self.calls()),2) | |
| self.assertTrue(all(c['state']=='settled' for c in self.calls())) | |
| def test_reset_routes_follow_verified_configuration_order(self): | |
| providers=[] | |
| for name in ['qwen3.8-flash','kimi-k3','deepseek-v4-flash-0731']: | |
| p=self.provider(lambda r:self.response(json.dumps({'files':SOURCE}))) | |
| p.model=name;p.routing_version=VERSION;p.selectable=True | |
| p.task_roles=['chat','design','code','repair'];p.health={'status':'healthy'};providers.append(p) | |
| run={**self.run,'snapshot':{'framework':{'dependencies':{'pdfbox':'fixed'}}}} | |
| # Quota burn-down: code tasks spend soonest-expiring quota first; unlisted models trail. | |
| self.assertEqual([p.model for p in plan(providers,'code',run)[0]], | |
| ['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash']) | |
| self.assertEqual([p.model for p in plan(providers,'chat',run)[0]], | |
| ['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash']) | |
| providers[0].health={'status':'rejected'} | |
| self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731','kimi-k3']) | |
| providers[1].health={'status':'not_probed'} | |
| self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731']) | |
| self.generate(providers,payload={'model':'attacker-model','endpoint':'https://attacker.example'}) | |
| self.assertEqual(self.calls()[0]['metadata']['model'],'deepseek-v4-flash-0731') | |
| def test_new_context_contract_has_explicit_ceiling_and_preserves_old_versions(self): | |
| from src.modules.skill.atomic_registry.catalog import builtin_operations,routing_operations,check_handler | |
| old=next(op for op in builtin_operations() if op.operation_id=='AO05') | |
| new=next(op for op in routing_operations() if op.operation_id=='AO05') | |
| check_handler(new) | |
| self.assertEqual(old.token_limit,20000);self.assertEqual(new.token_limit,32000) | |
| for invalid in (old.model_copy(update={'token_limit':32000}),new.model_copy(update={'token_limit':32001})): | |
| with self.assertRaises(SkillError):check_handler(invalid) | |
| self.assertEqual(self.runtime.store.get(self.run['id'])['release']['operations']['AO05@0.1.0']['token_limit'],20000) | |
| def test_compiler_reserves_the_larger_repair_context_from_the_same_budget(self): | |
| from src.modules.skill.atomic_registry.catalog import routing_operations | |
| from src.modules.skill.domain.models import Draft | |
| for op in routing_operations():self.runtime.skills.add_operation(AUTHOR,op) | |
| doc=self.runtime.skills.read_draft(AUTHOR,'erp-java-development') | |
| body={k:v for k,v in doc['body'].items() if k not in {'owner_id','last_editor_id'}} | |
| for node in body['graph']['nodes']: | |
| if node.get('operation_ref')=='AO05@0.1.0':node['operation_ref']='AO05@0.2.0' | |
| body['budget']=Budget(max_model_calls=6,max_tokens=110000,max_repairs=2).model_dump() | |
| changed=self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),doc['revision']) | |
| self.assertIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues]) | |
| body['budget']['max_tokens']=120000 | |
| self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),changed['revision']) | |
| self.assertNotIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues]) | |
| def test_default_and_existing_budgets_remain_unchanged_new_local_ceiling_is_bounded(self): | |
| self.assertEqual(self.runtime.store.get(self.run['id'])['budget']['max_model_calls'],3) | |
| self.assertEqual(Budget().max_tokens,60000) | |
| Budget(max_model_calls=6,max_tokens=120000,max_repairs=2) | |
| for data in ({'max_model_calls':7},{'max_tokens':120001}): | |
| with self.assertRaises(ValidationError):Budget(**data) | |
| def test_old_call_schema_migration_preserves_unknown_reservations(self): | |
| path=self.folder/'legacy.sqlite' | |
| with sqlite3.connect(path) as db: | |
| db.execute('CREATE TABLE calls(id INTEGER PRIMARY KEY,run_id TEXT,reservation INTEGER,charged INTEGER,state TEXT,provider TEXT,prompt_digest TEXT)') | |
| db.execute("INSERT INTO calls VALUES(1,'historical',100,NULL,'unknown','old','digest')") | |
| store=RuntimeStore(path) | |
| self.assertEqual(store.calls('historical')[0]['metadata'],{}) | |
| store.settle(1,'settled',0) | |
| self.assertEqual(store.calls('historical')[0]['state'],'unknown') | |
| self.assertIsNone(store.calls('historical')[0]['charged']) | |
| def test_repair_diagnostics_include_only_matching_failed_public_expectations(self): | |
| from types import SimpleNamespace | |
| self.runtime.sandbox.config=SimpleNamespace(tests=[SimpleNamespace(path='/contract',status=400,expected_json={'error':'INVALID_INPUT'})]) | |
| observed={'failures':[{'case_index':0,'path':'/contract','expected_status':400}, | |
| {'case_index':0,'path':'/different','expected_status':400}, | |
| {'case_index':50,'path':'/contract','expected_status':400}]} | |
| result=self.runtime.repair_diagnostics({'diagnostics':observed}) | |
| self.assertEqual(result['observed'],observed) | |
| self.assertEqual(result['public_contract_expectations'],[ | |
| {'case_index':0,'expected_status':400,'expected_json':{'error':'INVALID_INPUT'},'source':'server-public-contract'}]) | |
| self.assertNotIn('test_code',result) | |
| def test_calculator_repair_explains_disclosed_semantic_length_without_exposing_test_code(self): | |
| from types import SimpleNamespace | |
| from src.modules.execution.scenarios.calculator import calculation_path | |
| path=calculation_path('1+'*64+'1') | |
| case=SimpleNamespace(path=path,status=422,expected_json={'error':'LIMIT_EXCEEDED'}) | |
| self.runtime.sandbox.config=SimpleNamespace(tests=[case]) | |
| result=self.runtime.repair_diagnostics({'diagnostics':{'failures':[{'case_index':0,'path':path,'expected_status':422,'actual_status':400}]}}) | |
| item=result['public_contract_expectations'][0] | |
| self.assertEqual(item['decoded_request']['decoded_characters'],129) | |
| self.assertEqual(item['expected_status'],422) | |
| self.assertNotIn('test_code',result) | |
| if __name__=='__main__':unittest.main() | |