"""Protocol/fault tests with explicit synthetic providers; live Java evidence is separate.""" import json import sqlite3 import unittest from pathlib import Path import httpx from pydantic import ValidationError from src.adapters.llm.chat_completion import ChatCompletionProvider from src.adapters.persistence.runtime_sqlite import RuntimeStore from src.modules.llm_gateway.routing import VERSION,plan from src.modules.llm_gateway.service import LLMGateway,ModelReply,ProviderFailure from src.modules.skill.domain.models import Budget,SkillError from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE,AUTHOR class BailianGatewayTests(unittest.TestCase): def setUp(self): self.folder=Path(self.enterContext(workspace())) self.runtime,request,_,_=setup(self.folder) self.run=self.runtime.create(USER,request,'bailian-fault-test') self.runtime.store.mutate(self.run['id'],lambda r:r.update(status='running',claim='test')) def provider(self,handler,**options): return ChatCompletionProvider(name='synthetic-bailian',endpoint='https://model.example/v1/chat/completions', model='synthetic-approved-model',token='synthetic-secret',transport=httpx.MockTransport(handler),**options) def response(self,content,reason='stop',output=20): return httpx.Response(200,json={'id':'test-request-123','choices':[{'finish_reason':reason,'message':{'content':content}}], 'usage':{'prompt_tokens':10,'completion_tokens':output}}) def generate(self,providers,kind='code',cancelled=lambda:False,payload=None): return LLMGateway(self.runtime.store,providers).generate(self.run['id'],kind,payload or {},cancelled) def calls(self):return self.runtime.store.calls(self.run['id']) def test_normal_generation_is_one_call_with_larger_code_budget(self): seen=[] def respond(request): seen.append(json.loads(request.content)) return self.response(json.dumps({'files':SOURCE})) self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) self.assertEqual(len(seen),1) self.assertEqual(seen[0]['max_tokens'],8192) self.assertEqual(self.calls()[0]['charged'],30) self.assertEqual(self.calls()[0]['metadata']['request_id'],'test-request-123') def test_business_missing_page_is_known_invalid_and_regenerated_once(self): self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) seen=[] def respond(request): seen.append(request) files={'src/main/java/demo/App.java':'class App {}'} if len(seen)>1:files['website/index.html']='' return self.response(json.dumps({'files':files})) result=self.generate([self.provider(respond)]) self.assertEqual(set(result['files']),{'src/main/java/demo/App.java','website/index.html'}) calls=self.calls();self.assertEqual([c['state'] for c in calls],['settled','settled']) self.assertEqual(calls[0]['metadata']['error'],'MODEL_OUTPUT_INVALID') self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id']) def test_business_repair_can_correct_java_and_page_contract_together(self): self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) files={**SOURCE,'website/index.html':''} result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':files})))],kind='repair') self.assertEqual(result['files'],files) self.assertEqual(len(self.calls()),1) self.assertEqual(self.calls()[0]['state'],'settled') def test_explicit_java_only_continuation_still_retains_its_fixed_scope(self): def update(run): run['snapshot']['framework']['profile_id']='java17-requirement-business' run['request']['recovery_diagnostics']={'observed':'known Java failure'} self.runtime.store.mutate(self.run['id'],update) result=self.generate([self.provider(lambda request:self.response(json.dumps({'files':SOURCE})))]) self.assertEqual(result['files'],SOURCE) self.assertEqual(len(self.calls()),1) def test_business_embedded_page_is_rejected_and_regenerated(self): self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business')) seen=[] def respond(request): seen.append(request) java='class App { String page = ""; }' if len(seen)==1 else 'class App {}' return self.response(json.dumps({'files':{'src/main/java/demo/App.java':java,'website/index.html':''}})) result=self.generate([self.provider(respond)]) self.assertEqual(result['files']['src/main/java/demo/App.java'],'class App {}') self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_INVALID') def test_known_truncation_settles_usage_then_regenerates_complete_json(self): seen=[] def respond(request): seen.append(json.loads(request.content)) return (self.response('{"files":',reason='length',output=seen[-1]['max_tokens']) if len(seen)==1 else self.response(json.dumps({'files':SOURCE}))) self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) self.assertEqual(self.calls()[0]['charged'],8202) self.assertEqual(self.calls()[0]['metadata']['error'],'MODEL_OUTPUT_TRUNCATED') self.assertGreater(seen[1]['max_tokens'],seen[0]['max_tokens']) self.assertEqual(self.calls()[1]['metadata']['retry_of'],self.calls()[0]['id']) self.assertTrue(any(e['stage']=='model_recovery' for e in self.runtime.store.stages(self.run['id']))) def test_repeated_truncation_stops_and_remains_known(self): with self.assertRaises(SkillError) as err: self.generate([self.provider(lambda r:self.response('{',reason='length'))]) self.assertEqual(err.exception.code,'MODEL_OUTPUT_TRUNCATED') self.assertEqual([c['state'] for c in self.calls()],['settled','settled']) def test_invalid_json_and_invalid_schema_recover_only_once(self): for value in ('not json',json.dumps({'shell':'untrusted'})): with self.subTest(value=value): # Distinct run keys avoid treating two independent tests as retries. self.setUp() seen=[] def respond(request): seen.append(request) return self.response(value if len(seen)==1 else json.dumps({'files':SOURCE})) self.assertEqual(self.generate([self.provider(respond)])['files'],SOURCE) self.assertEqual(len(self.calls()),2) def test_unknown_transport_never_falls_back(self): seen=[] def timeout(request):raise httpx.ReadTimeout('synthetic-sensitive-body') backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE}))) with self.assertRaises(SkillError) as err:self.generate([self.provider(timeout),backup]) self.assertEqual(err.exception.code,'MODEL_RESULT_UNKNOWN') self.assertEqual(seen,[]) self.assertEqual(self.calls()[0]['state'],'unknown') self.assertIsNone(self.calls()[0]['charged']) self.assertEqual(self.calls()[0]['metadata']['failure_category'],'transport_timeout') diagnostics=self.runtime.store.diagnostics(self.run['id']) self.assertTrue(any(d['component']=='llm-gateway' and d['code']=='MODEL_RESULT_UNKNOWN' for d in diagnostics)) self.assertNotIn('synthetic-sensitive',json.dumps(diagnostics)) self.assertNotIn('synthetic-secret',json.dumps(diagnostics)) self.assertNotIn('synthetic-sensitive',json.dumps(self.calls())) def test_length_without_usage_is_unknown_not_retried(self): seen=[] def respond(request): seen.append(request) return httpx.Response(200,json={'choices':[{'finish_reason':'length','message':{'content':'{'}}]}) with self.assertRaises(SkillError):self.generate([self.provider(respond)]) self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'unknown') def test_quota_rejection_falls_back_but_authentication_does_not(self): for status,retry in [(429,True),(401,False)]: with self.subTest(status=status): self.setUp();seen=[] first=self.provider(lambda r:httpx.Response(status,text='synthetic raw error')) backup=self.provider(lambda r:seen.append(r) or self.response(json.dumps({'files':SOURCE}))) if retry:self.generate([first,backup]) else: with self.assertRaises(SkillError):self.generate([first,backup]) self.assertEqual(len(seen),int(retry));self.assertEqual(self.calls()[0]['charged'],0) def test_content_filter_and_tool_calls_are_known_but_not_replayed(self): for reason in ('content_filter','tool_calls'): with self.subTest(reason=reason): self.setUp() with self.assertRaises(SkillError) as err:self.generate([self.provider(lambda r:self.response('{}',reason=reason))]) self.assertEqual(err.exception.code,'MODEL_OUTPUT_UNSUPPORTED') self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['state'],'settled') def test_rejected_provider_code_is_retained_without_sensitive_message(self): provider=self.provider(lambda r:httpx.Response(403,json={'error':{ 'code':'AllocationQuota.FreeTierOnly','message':'quota exhausted; synthetic-secret account details'}})) with self.assertRaises(SkillError):self.generate([provider]) call=self.calls()[0] self.assertEqual(call['metadata']['provider_error_code'],'AllocationQuota.FreeTierOnly') self.assertEqual(call['state'],'rejected') self.assertEqual(call['charged'],0) self.assertNotIn('synthetic-secret',json.dumps(call)) self.assertTrue(call['metadata']['response_evidence']['response_received']) def test_recovery_obeys_total_call_limit(self): self.runtime.store.mutate(self.run['id'],lambda r:r['budget'].update(max_model_calls=1)) seen=[] def respond(r):seen.append(r);return self.response('{',reason='length') with self.assertRaises(SkillError) as err:self.generate([self.provider(respond)]) self.assertEqual(err.exception.code,'MODEL_CALL_BUDGET_EXCEEDED') self.assertEqual(len(seen),1) def test_stage_with_no_room_to_expand_does_not_repeat_a_truncated_response(self): seen=[] def respond(request): value=json.loads(request.content);seen.append(value) return self.response('{',reason='length',output=value['max_tokens']) gateway=LLMGateway(self.runtime.store,[self.provider(respond)]) with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,2000) self.assertEqual(err.exception.code,'MODEL_OUTPUT_LIMIT') self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled') def test_project_revocation_is_rechecked_before_recovery_dispatch(self): seen=[];checks=[] def guard(): checks.append(True) if len(checks)>1:raise SkillError('PROJECT_NOT_FOUND','revoked',404) def respond(request):seen.append(request);return self.response('{',reason='length') gateway=LLMGateway(self.runtime.store,[self.provider(respond)]) with self.assertRaises(SkillError) as err:gateway.generate(self.run['id'],'code',{},lambda:False,dispatch_guard=guard) self.assertEqual(err.exception.code,'PROJECT_NOT_FOUND') self.assertEqual(len(seen),1);self.assertEqual(self.calls()[0]['state'],'settled') def test_cancellation_after_response_settles_and_never_retries(self): state={'cancelled':False} class Model: name='synthetic-cancelled' def complete(self,*args): state['cancelled']=True raise ProviderFailure('MODEL_OUTPUT_TRUNCATED',input_tokens=10,output_tokens=10) with self.assertRaises(SkillError) as err:self.generate([Model()],cancelled=lambda:state['cancelled']) self.assertEqual(err.exception.code,'RUN_CANCELLED') self.assertEqual(len(self.calls()),1);self.assertEqual(self.calls()[0]['charged'],20) def test_coder_profile_does_not_send_unsupported_json_or_thinking_parameters(self): def respond(request): value=json.loads(request.content) self.assertNotIn('response_format',value);self.assertNotIn('enable_thinking',value) return self.response(json.dumps({'files':SOURCE})) self.generate([self.provider(respond,json_mode=False)]) self.assertNotIn('synthetic-secret',json.dumps(self.calls())) self.assertNotIn('synthetic-secret',json.dumps(LLMGateway(self.runtime.store,[self.provider(respond)]).routes())) def test_known_invalid_review_recovers_once_and_keeps_safe_validation_reason(self): seen=[] def respond(request): seen.append(json.loads(request.content)) body={'summary':'review','findings':[{'secret':'sensitive-untrusted-value'}] if len(seen)==1 else [],'files':SOURCE} return self.response(json.dumps(body)) result=self.generate([self.provider(respond)],kind='code_review',payload={'patch':{'files':SOURCE}}) self.assertEqual(result['files'],SOURCE) calls=self.calls();self.assertEqual(len(calls),2) self.assertEqual([c['state'] for c in calls],['settled','settled']) self.assertEqual(calls[1]['metadata']['retry_of'],calls[0]['id']) self.assertEqual(calls[0]['metadata']['validation_errors'],[{'location':['findings',0],'type':'string_type'}]) self.assertNotIn('sensitive-untrusted-value',json.dumps(calls)) def test_repeated_invalid_review_stops_within_shared_call_budget(self): with self.assertRaises(SkillError) as error: self.generate([self.provider(lambda r:self.response(json.dumps({'files':SOURCE})))],kind='code_review',payload={'patch':{'files':SOURCE}}) self.assertEqual(error.exception.code,'MODEL_OUTPUT_INVALID') self.assertEqual(len(self.calls()),2) self.assertTrue(all(c['state']=='settled' for c in self.calls())) def test_reset_routes_follow_verified_configuration_order(self): providers=[] for name in ['qwen3.8-flash','kimi-k3','deepseek-v4-flash-0731']: p=self.provider(lambda r:self.response(json.dumps({'files':SOURCE}))) p.model=name;p.routing_version=VERSION;p.selectable=True p.task_roles=['chat','design','code','repair'];p.health={'status':'healthy'};providers.append(p) run={**self.run,'snapshot':{'framework':{'dependencies':{'pdfbox':'fixed'}}}} # Quota burn-down: code tasks spend soonest-expiring quota first; unlisted models trail. self.assertEqual([p.model for p in plan(providers,'code',run)[0]], ['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash']) self.assertEqual([p.model for p in plan(providers,'chat',run)[0]], ['deepseek-v4-flash-0731','kimi-k3','qwen3.8-flash']) providers[0].health={'status':'rejected'} self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731','kimi-k3']) providers[1].health={'status':'not_probed'} self.assertEqual([p.model for p in plan(providers,'code',run)[0]],['deepseek-v4-flash-0731']) self.generate(providers,payload={'model':'attacker-model','endpoint':'https://attacker.example'}) self.assertEqual(self.calls()[0]['metadata']['model'],'deepseek-v4-flash-0731') def test_new_context_contract_has_explicit_ceiling_and_preserves_old_versions(self): from src.modules.skill.atomic_registry.catalog import builtin_operations,routing_operations,check_handler old=next(op for op in builtin_operations() if op.operation_id=='AO05') new=next(op for op in routing_operations() if op.operation_id=='AO05') check_handler(new) self.assertEqual(old.token_limit,20000);self.assertEqual(new.token_limit,32000) for invalid in (old.model_copy(update={'token_limit':32000}),new.model_copy(update={'token_limit':32001})): with self.assertRaises(SkillError):check_handler(invalid) self.assertEqual(self.runtime.store.get(self.run['id'])['release']['operations']['AO05@0.1.0']['token_limit'],20000) def test_compiler_reserves_the_larger_repair_context_from_the_same_budget(self): from src.modules.skill.atomic_registry.catalog import routing_operations from src.modules.skill.domain.models import Draft for op in routing_operations():self.runtime.skills.add_operation(AUTHOR,op) doc=self.runtime.skills.read_draft(AUTHOR,'erp-java-development') body={k:v for k,v in doc['body'].items() if k not in {'owner_id','last_editor_id'}} for node in body['graph']['nodes']: if node.get('operation_ref')=='AO05@0.1.0':node['operation_ref']='AO05@0.2.0' body['budget']=Budget(max_model_calls=6,max_tokens=110000,max_repairs=2).model_dump() changed=self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),doc['revision']) self.assertIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues]) body['budget']['max_tokens']=120000 self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),changed['revision']) self.assertNotIn('BUDGET_UNREACHABLE',[e.code for e in self.runtime.skills.validate(AUTHOR,'erp-java-development').issues]) def test_default_and_existing_budgets_remain_unchanged_new_local_ceiling_is_bounded(self): self.assertEqual(self.runtime.store.get(self.run['id'])['budget']['max_model_calls'],3) self.assertEqual(Budget().max_tokens,60000) Budget(max_model_calls=6,max_tokens=120000,max_repairs=2) for data in ({'max_model_calls':7},{'max_tokens':120001}): with self.assertRaises(ValidationError):Budget(**data) def test_old_call_schema_migration_preserves_unknown_reservations(self): path=self.folder/'legacy.sqlite' with sqlite3.connect(path) as db: db.execute('CREATE TABLE calls(id INTEGER PRIMARY KEY,run_id TEXT,reservation INTEGER,charged INTEGER,state TEXT,provider TEXT,prompt_digest TEXT)') db.execute("INSERT INTO calls VALUES(1,'historical',100,NULL,'unknown','old','digest')") store=RuntimeStore(path) self.assertEqual(store.calls('historical')[0]['metadata'],{}) store.settle(1,'settled',0) self.assertEqual(store.calls('historical')[0]['state'],'unknown') self.assertIsNone(store.calls('historical')[0]['charged']) def test_repair_diagnostics_include_only_matching_failed_public_expectations(self): from types import SimpleNamespace self.runtime.sandbox.config=SimpleNamespace(tests=[SimpleNamespace(path='/contract',status=400,expected_json={'error':'INVALID_INPUT'})]) observed={'failures':[{'case_index':0,'path':'/contract','expected_status':400}, {'case_index':0,'path':'/different','expected_status':400}, {'case_index':50,'path':'/contract','expected_status':400}]} result=self.runtime.repair_diagnostics({'diagnostics':observed}) self.assertEqual(result['observed'],observed) self.assertEqual(result['public_contract_expectations'],[ {'case_index':0,'expected_status':400,'expected_json':{'error':'INVALID_INPUT'},'source':'server-public-contract'}]) self.assertNotIn('test_code',result) def test_calculator_repair_explains_disclosed_semantic_length_without_exposing_test_code(self): from types import SimpleNamespace from src.modules.execution.scenarios.calculator import calculation_path path=calculation_path('1+'*64+'1') case=SimpleNamespace(path=path,status=422,expected_json={'error':'LIMIT_EXCEEDED'}) self.runtime.sandbox.config=SimpleNamespace(tests=[case]) result=self.runtime.repair_diagnostics({'diagnostics':{'failures':[{'case_index':0,'path':path,'expected_status':422,'actual_status':400}]}}) item=result['public_contract_expectations'][0] self.assertEqual(item['decoded_request']['decoded_characters'],129) self.assertEqual(item['expected_status'],422) self.assertNotIn('test_code',result) if __name__=='__main__':unittest.main()