VibeCoding / src /tests /runtime /test_bailian_gateway.py
dellweb3's picture
sol6.1 ERP稳定版本
115b94d verified
Raw History Blame Contribute Delete
21.5 kB
"""Protocol/fault tests with explicit synthetic providers; live Java evidence is separate."""
import json
import sqlite3
import unittest
from pathlib import Path
import httpx
from pydantic import ValidationError
from src.adapters.llm.chat_completion import ChatCompletionProvider
from src.adapters.persistence.runtime_sqlite import RuntimeStore
from src.modules.llm_gateway.routing import VERSION,plan
from src.modules.llm_gateway.service import LLMGateway,ModelReply,ProviderFailure
from src.modules.skill.domain.models import Budget,SkillError
from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE,AUTHOR
class BailianGatewayTests(unittest.TestCase):
def setUp(self):
self.folder=Path(self.enterContext(workspace()))
self.runtime,request,_,_=setup(self.folder)
self.run=self.runtime.create(USER,request,'bailian-fault-test')
self.runtime.store.mutate(self.run['id'],lambda r:r.update(status='running',claim='test'))
def provider(self,handler,**options):
return ChatCompletionProvider(name='synthetic-bailian',endpoint='https://model.example/v1/chat/completions',
model='synthetic-approved-model',token='synthetic-secret',transport=httpx.MockTransport(handler),**options)
def response(self,content,reason='stop',output=20):
return httpx.Response(200,json={'id':'test-request-123','choices':[{'finish_reason':reason,'message':{'content':content}}],
'usage':{'prompt_tokens':10,'completion_tokens':output}})
def generate(self,providers,kind='code',cancelled=lambda:False,payload=None):
return LLMGateway(self.runtime.store,providers).generate(self.run['id'],kind,payload or {},cancelled)
def calls(self):return self.runtime.store.calls(self.run['id'])
def test_normal_generation_is_one_call_with_larger_code_budget(self):
seen=[]
def respond(request):
seen.append(json.loads(request.content))
return self.response(json.dumps({'files':SOURCE}))
self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE)
self.assertEqual(len(seen),1)
self.assertEqual(seen[0]['max_tokens'],8192)
self.assertEqual(self.calls()[0]['charged'],30)
self.assertEqual(self.calls()[0]['metadata']['request_id'],'test-request-123')
def test_business_missing_page_is_known_invalid_and_regenerated_once(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business'))
seen=[]
def respond(request):
seen.append(request)
files={'src/main/java/demo/App.java':'class App {}'}
if len(seen)>1:files['website/index.html']='<html><style></style><script></script></html>'
return self.response(json.dumps({'files':files}))
result=self.generate([self.provider(respond)])
self.assertEqual(set(result['files']),{'src/main/java/demo/App.java','website/index.html'})
calls=self.calls();self.assertEqual([c['state'] for c in calls],['settled','settled'])
self.assertEqual(calls[0]['metadata']['error'],'MODEL_OUTPUT_INVALID')
self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id'])
def test_business_repair_can_correct_java_and_page_contract_together(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business'))
files={**SOURCE,'website/index.html':'<html><script>const rows=data.items;</script></html>'}
result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':files})))],kind='repair')
self.assertEqual(result['files'],files)
self.assertEqual(len(self.calls()),1)
self.assertEqual(self.calls()[0]['state'],'settled')
def test_explicit_java_only_continuation_still_retains_its_fixed_scope(self):
def update(run):
run['snapshot']['framework']['profile_id']='java17-requirement-business'
run['request']['recovery_diagnostics']={'observed':'known Java failure'}
self.runtime.store.mutate(self.run['id'],update)
result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':SOURCE})))])
self.assertEqual(result['files'],SOURCE)
self.assertEqual(len(self.calls()),1)
def test_business_embedded_page_is_rejected_and_regenerated(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business'))
seen=[]
def respond(request):
seen.append(request)
java='class App { String page = "<html>"; }' if len(seen)==1 else 'class App {}'
return self.response(json.dumps({'files':{'src/main/java/demo/App.java':java,'website/index.html':'<html></html>'}}))
result=self.generate([self.provider(respond)])
self.assertEqual(result['files']['src/main/java/demo/App.java'],'class App {}')
self.assertEqual([c['state'] for c in self.calls()],['settled','settled'])
self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_INVALID')
def test_known_truncation_settles_usage_then_regenerates_complete_json(self):
seen=[]
def respond(request):
seen.append(json.loads(request.content))
return (self.response('{"files":',reason='length',output=seen[-1]['max_tokens']) if len(seen)==1
else self.response(json.dumps({'files':SOURCE})))
self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE)
self.assertEqual([c['state'] for c in self.calls()],['settled','settled'])
self.assertEqual(self.calls()[0]['charged'],8202)
self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_TRUNCATED')
self.assertGreater(seen[1]['max_tokens'],seen[0]['max_tokens'])
self.assertEqual(self.calls()[1]['metadata']['retry_of'],self.calls()[0]['id'])
self.assertTrue(any(e['stage']=='model_recovery' for e in self.runtime.store.stages(self.run['id'])))
def test_repeated_truncation_stops_and_remains_known(self):
with self.assertRaises(SkillError) as err:
self.generate([self.provider(lambda r:self.response('{',reason='length'))])
self.assertEqual(err.exception.code,'MODEL_OUTPUT_TRUNCATED')
self.assertEqual([c['state'] for c in self.calls()],['settled','settled'])
def test_invalid_json_and_invalid_schema_recover_only_once(self):
for value in ('not json',json.dumps({'shell':'untrusted'})):
with self.subTest(value=value):
# Distinct run keys avoid treating two independent tests as retries.
self.setUp()
seen=[]
def respond(request):
seen.append(request)
return self.response(value if len(seen)==1 else json.dumps({'files':SOURCE}))
self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE)
self.assertEqual(len(self.calls()),2)
def test_unknown_transport_never_falls_back(self):
seen=[]
def timeout(request):raise httpx.ReadTimeout('synthetic-sensitive-body')
backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE})))
with self.assertRaises(SkillError) as err:self.generate([self.provider(timeout),backup])
self.assertEqual(err.exception.code,'MODEL_RESULT_UNKNOWN')
self.assertEqual(seen,[])
self.assertEqual(self.calls()[0]['state'],'unknown')
self.assertIsNone(self.calls()[0]['charged'])
self.assertEqual(self.calls()[0]['metadata']['failure_category'],'transport_timeout')
diagnostics=self.runtime.store.diagnostics(self.run['id'])
self.assertTrue(any(d['component']=='llm-gateway' and d['code']=='MODEL_RESULT_UNKNOWN' for d in diagnostics))
self.assertNotIn('synthetic-sensitive',json.dumps(diagnostics))
self.assertNotIn('synthetic-secret',json.dumps(diagnostics))
self.assertNotIn('synthetic-sensitive',json.dumps(self.calls()))
def test_length_without_usage_is_unknown_not_retried(self):
seen=[]
def respond(request):
seen.append(request)
return httpx.Response(200,json={'choices':[{'finish_reason':'length','message':{'content':'{'}}]})
with self.assertRaises(SkillError):self.generate([self.provider(respond)])
self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'unknown')
def test_quota_rejection_falls_back_but_authentication_does_not(self):
for status,retry in [(429,True),(401,False)]:
with self.subTest(status=status):
self.setUp();seen=[]
first=self.provider(lambda r:httpx.Response(status,text='synthetic raw error'))
backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE})))
if retry:self.generate([first,backup])
else:
with self.assertRaises(SkillError):self.generate([first,backup])
self.assertEqual(len(seen),int(retry));self.assertEqual(self.calls()[0]['charged'],0)
def test_content_filter_and_tool_calls_are_known_but_not_replayed(self):
for reason in ('content_filter','tool_calls'):
with self.subTest(reason=reason):
self.setUp()
with self.assertRaises(SkillError) as err:self.generate([self.provider(lambda r:self.response('{}',reason=reason))])
self.assertEqual(err.exception.code,'MODEL_OUTPUT_UNSUPPORTED')
self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['state'],'settled')
def test_rejected_provider_code_is_retained_without_sensitive_message(self):
provider=self.provider(lambda r:httpx.Response(403,json={'error':{
'code':'AllocationQuota.FreeTierOnly','message':'quota exhausted; synthetic-secret account details'}}))
with self.assertRaises(SkillError):self.generate([provider])
call=self.calls()[0]
self.assertEqual(call['metadata']['provider_error_code'],'AllocationQuota.FreeTierOnly')
self.assertEqual(call['state'],'rejected')
self.assertEqual(call['charged'],0)
self.assertNotIn('synthetic-secret',json.dumps(call))
self.assertTrue(call['metadata']['response_evidence']['response_received'])
def test_recovery_obeys_total_call_limit(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['budget'].update(max_model_calls=1))
seen=[]
def respond(r):seen.append(r);return self.response('{',reason='length')
with self.assertRaises(SkillError) as err:self.generate([self.provider(respond)])
self.assertEqual(err.exception.code,'MODEL_CALL_BUDGET_EXCEEDED')
self.assertEqual(len(seen),1)
def test_stage_with_no_room_to_expand_does_not_repeat_a_truncated_response(self):
seen=[]
def respond(request):
value=json.loads(request.content);seen.append(value)
return self.response('{',reason='length',output=value['max_tokens'])
gateway=LLMGateway(self.runtime.store,[self.provider(respond)])
with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,2000)
self.assertEqual(err.exception.code,'MODEL_OUTPUT_LIMIT')
self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled')
def test_project_revocation_is_rechecked_before_recovery_dispatch(self):
seen=[];checks=[]
def guard():
checks.append(True)
if len(checks)>1:raise SkillError('PROJECT_NOT_FOUND','revoked',404)
def respond(request):seen.append(request);return self.response('{',reason='length')
gateway=LLMGateway(self.runtime.store,[self.provider(respond)])
with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,dispatch_guard=guard)
self.assertEqual(err.exception.code,'PROJECT_NOT_FOUND')
self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled')
def test_cancellation_after_response_settles_and_never_retries(self):
state={'cancelled':False}
class Model:
name='synthetic-cancelled'
def complete(self,*args):
state['cancelled']=True
raise ProviderFailure('MODEL_OUTPUT_TRUNCATED',input_tokens=10,output_tokens=10)
with self.assertRaises(SkillError) as err:self.generate([Model()],cancelled=lambda:state['cancelled'])
self.assertEqual(err.exception.code,'RUN_CANCELLED')
self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['charged'],20)
def test_coder_profile_does_not_send_unsupported_json_or_thinking_parameters(self):
def respond(request):
value=json.loads(request.content)
self.assertNotIn('response_format',value);self.assertNotIn('enable_thinking',value)
return self.response(json.dumps({'files':SOURCE}))
self.generate([self.provider(respond,json_mode=False)])
self.assertNotIn('synthetic-secret',json.dumps(self.calls()))
self.assertNotIn('synthetic-secret',json.dumps(LLMGateway(self.runtime.store,[self.provider(respond)]).routes()))
def test_known_invalid_review_recovers_once_and_keeps_safe_validation_reason(self):
seen=[]
def respond(request):
seen.append(json.loads(request.content))
body={'summary':'review','findings':[{'secret':'sensitive-untrusted-value'}] if len(seen)==1 else [],'files':SOURCE}
return self.response(json.dumps(body))
result=self.generate([self.provider(respond)],kind='code_review',payload={'patch':{'files':SOURCE}})
self.assertEqual(result['files'],SOURCE)
calls=self.calls();self.assertEqual(len(calls),2)
self.assertEqual([c['state'] for c in calls],['settled','settled'])
self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id'])
self.assertEqual(calls[0]['metadata']['validation_errors'],[{'location':['findings',0],'type':'string_type'}])
self.assertNotIn('sensitive-untrusted-value',json.dumps(calls))
def test_repeated_invalid_review_stops_within_shared_call_budget(self):
with self.assertRaises(SkillError) as error:
self.generate([self.provider(lambda r:self.response(json.dumps({'files':SOURCE})))],kind='code_review',payload={'patch':{'files':SOURCE}})
self.assertEqual(error.exception.code,'MODEL_OUTPUT_INVALID')
self.assertEqual(len(self.calls()),2)
self.assertTrue(all(c['state']=='settled' for c in self.calls()))
def test_reset_routes_follow_verified_configuration_order(self):
providers=[]
for name in ['qwen3.8-flash','kimi-k3','deepseek-v4-flash-0731']:
p=self.provider(lambda r:self.response(json.dumps({'files':SOURCE})))
p.model=name;p.routing_version=VERSION;p.selectable=True
p.task_roles=['chat','design','code','repair'];p.health={'status':'healthy'};providers.append(p)
run={**self.run,'snapshot':{'framework':{'dependencies':{'pdfbox':'fixed'}}}}
# Quota burn-down: code tasks spend soonest-expiring quota first; unlisted models trail.
self.assertEqual([p.model for p in plan(providers,'code',run)[0]],
['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash'])
self.assertEqual([p.model for p in plan(providers,'chat',run)[0]],
['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash'])
providers[0].health={'status':'rejected'}
self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731','kimi-k3'])
providers[1].health={'status':'not_probed'}
self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731'])
self.generate(providers,payload={'model':'attacker-model','endpoint':'https://attacker.example'})
self.assertEqual(self.calls()[0]['metadata']['model'],'deepseek-v4-flash-0731')
def test_new_context_contract_has_explicit_ceiling_and_preserves_old_versions(self):
from src.modules.skill.atomic_registry.catalog import builtin_operations,routing_operations,check_handler
old=next(op for op in builtin_operations() if op.operation_id=='AO05')
new=next(op for op in routing_operations() if op.operation_id=='AO05')
check_handler(new)
self.assertEqual(old.token_limit,20000);self.assertEqual(new.token_limit,32000)
for invalid in (old.model_copy(update={'token_limit':32000}),new.model_copy(update={'token_limit':32001})):
with self.assertRaises(SkillError):check_handler(invalid)
self.assertEqual(self.runtime.store.get(self.run['id'])['release']['operations']['AO05@0.1.0']['token_limit'],20000)
def test_compiler_reserves_the_larger_repair_context_from_the_same_budget(self):
from src.modules.skill.atomic_registry.catalog import routing_operations
from src.modules.skill.domain.models import Draft
for op in routing_operations():self.runtime.skills.add_operation(AUTHOR,op)
doc=self.runtime.skills.read_draft(AUTHOR,'erp-java-development')
body={k:v for k,v in doc['body'].items() if k not in {'owner_id','last_editor_id'}}
for node in body['graph']['nodes']:
if node.get('operation_ref')=='AO05@0.1.0':node['operation_ref']='AO05@0.2.0'
body['budget']=Budget(max_model_calls=6,max_tokens=110000,max_repairs=2).model_dump()
changed=self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),doc['revision'])
self.assertIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues])
body['budget']['max_tokens']=120000
self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),changed['revision'])
self.assertNotIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues])
def test_default_and_existing_budgets_remain_unchanged_new_local_ceiling_is_bounded(self):
self.assertEqual(self.runtime.store.get(self.run['id'])['budget']['max_model_calls'],3)
self.assertEqual(Budget().max_tokens,60000)
Budget(max_model_calls=6,max_tokens=120000,max_repairs=2)
for data in ({'max_model_calls':7},{'max_tokens':120001}):
with self.assertRaises(ValidationError):Budget(**data)
def test_old_call_schema_migration_preserves_unknown_reservations(self):
path=self.folder/'legacy.sqlite'
with sqlite3.connect(path) as db:
db.execute('CREATE TABLE calls(id INTEGER PRIMARY KEY,run_id TEXT,reservation INTEGER,charged INTEGER,state TEXT,provider TEXT,prompt_digest TEXT)')
db.execute("INSERT INTO calls VALUES(1,'historical',100,NULL,'unknown','old','digest')")
store=RuntimeStore(path)
self.assertEqual(store.calls('historical')[0]['metadata'],{})
store.settle(1,'settled',0)
self.assertEqual(store.calls('historical')[0]['state'],'unknown')
self.assertIsNone(store.calls('historical')[0]['charged'])
def test_repair_diagnostics_include_only_matching_failed_public_expectations(self):
from types import SimpleNamespace
self.runtime.sandbox.config=SimpleNamespace(tests=[SimpleNamespace(path='/contract',status=400,expected_json={'error':'INVALID_INPUT'})])
observed={'failures':[{'case_index':0,'path':'/contract','expected_status':400},
{'case_index':0,'path':'/different','expected_status':400},
{'case_index':50,'path':'/contract','expected_status':400}]}
result=self.runtime.repair_diagnostics({'diagnostics':observed})
self.assertEqual(result['observed'],observed)
self.assertEqual(result['public_contract_expectations'],[
{'case_index':0,'expected_status':400,'expected_json':{'error':'INVALID_INPUT'},'source':'server-public-contract'}])
self.assertNotIn('test_code',result)
def test_calculator_repair_explains_disclosed_semantic_length_without_exposing_test_code(self):
from types import SimpleNamespace
from src.modules.execution.scenarios.calculator import calculation_path
path=calculation_path('1+'*64+'1')
case=SimpleNamespace(path=path,status=422,expected_json={'error':'LIMIT_EXCEEDED'})
self.runtime.sandbox.config=SimpleNamespace(tests=[case])
result=self.runtime.repair_diagnostics({'diagnostics':{'failures':[{'case_index':0,'path':path,'expected_status':422,'actual_status':400}]}})
item=result['public_contract_expectations'][0]
self.assertEqual(item['decoded_request']['decoded_characters'],129)
self.assertEqual(item['expected_status'],422)
self.assertNotIn('test_code',result)
if __name__=='__main__':unittest.main()