VibeCoding / src /tests /runtime /test_repair_budget.py
dellweb3's picture
sol6.1 ERP稳定版本
115b94d verified
Raw History Blame Contribute Delete
17.1 kB
"""Budget boundaries and compiler edits; synthetic protocol, no paid calls."""
import json
import unittest
import httpx
from types import SimpleNamespace
from src.tests.runtime import test_bailian_gateway as gateway_fixtures
from src.modules.llm_gateway.service import LLMGateway,ProviderFailure
from src.modules.context.service import ContextService
from src.modules.orchestrator.service import AgentRuntime
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup,workspace,USER,OTHER
class PreflightTests(unittest.TestCase):
def check(self,stage=32000,total=120000,calls=(),window=None,text='x'):
return LLMGateway.preflight([{'content':'system'},{'content':text}],{'requirement':text},
{'budget':{'max_tokens':total}},calls,stage,12288,SimpleNamespace(context_window=window),'repair')[1]
def test_normal_keeps_safety_and_output_reservation(self):
c=self.check();self.assertIsNone(c['rejection'])
self.assertEqual(c['input_bound'],519)
self.assertEqual(c['output_reservation'],12288)
self.assertFalse(c['model_context_limit_verified'])
def test_distinct_stage_task_and_model_rejections(self):
for options,code in [({'stage':600},'STAGE_CONTEXT_BUDGET_EXCEEDED'),
({'total':600},'TASK_TOKEN_BUDGET_EXCEEDED'),
({'window':600},'MODEL_CONTEXT_LIMIT_EXCEEDED')]:
with self.subTest(code=code):self.assertEqual(self.check(**options)['rejection'][0],code)
def test_exact_output_floor_and_unknown_reservations_count(self):
self.assertIsNone(self.check(stage=647)['rejection'])
self.assertEqual(self.check(stage=646)['rejection'][0],'STAGE_CONTEXT_BUDGET_EXCEEDED')
c=self.check(total=1000,calls=[{'charged':None,'reservation':400}])
self.assertEqual(c['reserved_tokens'],400)
self.assertEqual(c['task_remaining'],600)
self.assertEqual(c['rejection'][0],'TASK_TOKEN_BUDGET_EXCEEDED')
def test_compiler_scope_keeps_full_java_and_hashes_unchanged_page(self):
files={'src/main/java/demo/App.java':'complete Java','website/index.html':'complete page'}
diag={'diagnostics':'/input/src/main/java/demo/App.java:268: error: duplicate variable\ncode\n ^'}
inputs=ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',diag)
self.assertEqual(inputs['previous'],{'src/main/java/demo/App.java':'complete Java'})
self.assertIn('website/index.html',inputs['repair_scope']['unchanged_files'])
for code in ['JAVA_TEST_FAILED','MODEL_OUTPUT_INVALID']:
self.assertEqual(ContextService.repair_inputs(files,code,diag)['previous'],files)
self.assertEqual(ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',{})['previous'],files)
def test_list_failure_keeps_both_logic_views_and_explicit_omission_records(self):
files={'src/main/java/demo/App.java':' static void generateInitialData() { /* unrelated */ }\n static void handleList() { /* relevant */ }\n',
'website/index.html':'<style>retained</style><button id="next">Next</button><script>function load(){return "records";}</script>'}
data=ContextService.repair_inputs(files,'JAVA_TEST_FAILED',{'failures':[{'path':'/api/app/list'}]})
self.assertEqual(data['repair_scope']['mode'],'exact-edits')
self.assertIn('handleList',data['previous']['src/main/java/demo/App.java'])
self.assertIn('load()',data['previous']['website/index.html'])
self.assertEqual(data['repair_scope']['omitted_compiled_methods'][0]['method'],'generateInitialData')
self.assertIn('id="next"',data['page_elements'][0])
self.assertIn('<style>retained</style>',files['website/index.html'])
class BudgetConsentTests(unittest.TestCase):
def test_repair_resume_retains_original_failure_and_identical_context(self):
from src.modules.llm_gateway.service import ModelReply
with workspace() as folder:
rt,request,model,_=setup(folder)
request={**request,'requirement':'帮我开发一个物流管理的程序,作为物流公司,能够检索到当前订单的状态,并根据实际的物流进行状态调整。'}
run=rt.create(USER,request,'logistics-repair-resume')
java='src/main/java/demo/App.java'
files={java:'package demo; public class App { int duplicate=0; int duplicate=1; public static void main(String[]a){} } /*'+('x'*24000)+'*/'}
def prepare(r):
nodes=r['release']['draft']['graph']['nodes']
node=lambda op:next(n['id'] for n in nodes if (n.get('operation_ref') or '').startswith(op+'@'))
r.update(repair_pending=True,repairs=1,error='JAVA_BUILD_FAILED',
diagnostics={'diagnostics':'/input/src/main/java/demo/App.java:1: error: variable duplicate is already defined'},
outputs={node('AO04'):{'design':{'summary':'固定原始物流需求,不扩展模块。'}},node('AO06'):{'source':{'files':files}}})
r['budget']['max_tokens']=120000
rt.store.mutate(run['id'],prepare)
def complete(messages,bound,cancelled):
payload=json.loads(messages[-1]['content'])['data']
self.assertEqual(payload['failure_code'],'JAVA_BUILD_FAILED')
self.assertTrue(payload['repair_scope']['compiler_only'])
model.count+=1
return ModelReply({'edits':[{'old':'int duplicate=0; int duplicate=1;','new':'int duplicate=0; int fixed=1;'}]},100,20,model.name)
model.complete=complete
paused=rt.advance(USER,run['id']);self.assertEqual(paused['status'],'waiting_budget')
p=paused['budget_request'];self.assertEqual(paused['repair_failure_code'],'JAVA_BUILD_FAILED')
rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
resumed=rt.advance(USER,run['id'])
self.assertEqual(resumed['status'],'waiting_approval');self.assertEqual(model.count,1)
self.assertEqual(len(rt.store.calls(run['id'])),1)
self.assertEqual(rt.store.calls(run['id'])[0]['metadata']['budget_check']['input_bound'],p['input_bound'])
def paused(self,folder):
runtime,request,model,_=setup(folder)
run=runtime.create(USER,request,'budget-consent-001')
runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_tokens=100))
run=runtime.advance(USER,run['id'])
self.assertEqual(run['status'],'waiting_budget')
self.assertEqual(model.count,0)
self.assertEqual(runtime.store.calls(run['id']),[])
return runtime,run,model
def test_explicit_approval_resumes_same_stage_and_preserves_snapshot(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
pending=run['budget_request'];snapshot=run['snapshot']
with self.assertRaises(SkillError):rt.decide_budget(OTHER,run['id'],pending['digest'],True)
with self.assertRaises(SkillError):rt.decide_budget(USER,run['id'],'0'*64,True)
approved=rt.decide_budget(USER,run['id'],pending['digest'],True)
self.assertEqual(approved['id'],run['id'])
self.assertEqual(approved['snapshot'],snapshot)
self.assertEqual(approved['budget']['max_tokens'],pending['task_limit'])
duplicate=rt.decide_budget(USER,run['id'],pending['digest'],True)
self.assertFalse(duplicate['budget_decision_applied'])
self.assertEqual(duplicate['revision'],approved['revision'])
resumed=rt.advance(USER,run['id'])
self.assertEqual(resumed['status'],'waiting_approval')
self.assertEqual(model.count,1)
self.assertEqual(len(resumed['budget_authorizations']),1)
def test_declining_preserves_evidence_and_never_calls_model(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
result=rt.decide_budget(USER,run['id'],run['budget_request']['digest'],False)
self.assertEqual(result['status'],'waiting_budget')
self.assertTrue(result['budget_request']['deferred'])
self.assertEqual(result['budget']['max_tokens'],100)
self.assertTrue(result['budget_checks'])
self.assertEqual(model.count,0)
def test_expiry_recalculates_and_requires_a_new_confirmation(self):
with workspace() as folder:
rt,run,model=self.paused(folder);pending=run['budget_request']
rt.clock=lambda:pending['expires_at']+1
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],pending['digest'],True,pending['task_revision'])
self.assertEqual(err.exception.code,'BUDGET_REQUEST_CHANGED')
fresh=rt.store.get(run['id'])
self.assertNotEqual(fresh['budget_request']['digest'],pending['digest'])
self.assertEqual(fresh['budget']['max_tokens'],100);self.assertEqual(model.count,0)
rt.decide_budget(USER,run['id'],fresh['budget_request']['digest'],True,fresh['budget_request']['task_revision'])
self.assertEqual(model.count,0) # Decision enqueues only; no direct dispatch.
def test_changed_task_version_or_context_never_uses_stale_approval(self):
for context_changed in (False,True):
with self.subTest(context_changed=context_changed),workspace() as folder:
rt,run,model=self.paused(folder);p=run['budget_request']
rt.store.mutate(run['id'],lambda r:r['outputs'].update(changed=True) if context_changed else r.update(audit_marker=True))
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
self.assertEqual(err.exception.code,'BUDGET_REQUEST_CHANGED')
changed=rt.store.get(run['id']);self.assertEqual(changed['budget']['max_tokens'],100)
self.assertEqual(model.count,0)
self.assertEqual(changed['status'],'failed' if context_changed else 'waiting_budget')
def test_model_window_cannot_be_overridden_by_approval(self):
with workspace() as folder:
rt,run,model=self.paused(folder);p=run['budget_request']
model.context_window=1
with self.assertRaises(SkillError):rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
refused=rt.store.get(run['id']);self.assertEqual(refused['error'],'MODEL_CONTEXT_LIMIT_EXCEEDED')
self.assertIsNone(refused['budget_request']);self.assertEqual(model.count,0)
def test_environment_and_supplier_refusals_never_create_budget_approval(self):
for code in ('SANDBOX_BOOT_FAILED','JAVA_TEST_FAILED','MODEL_QUOTA_REJECTED'):
with self.subTest(code=code),workspace() as folder:
rt,request,model,_=setup(folder);run=rt.create(USER,request,'not-budget-'+code)
def fail(*a,**kw):raise SkillError(code,'synthetic fault',409)
rt.llm.generate=fail;out=rt.advance(USER,run['id'])
self.assertNotEqual(out['status'],'waiting_budget');self.assertIsNone(out.get('budget_request'))
self.assertEqual(model.count,0)
def test_approval_cannot_release_unknown_reservation(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
rt.store.mutate(run['id'],lambda r:r.update(status='running'))
call=rt.store.reserve(run['id'],'synthetic',[],50)
rt.store.settle(call,'unknown')
rt.store.mutate(run['id'],lambda r:r.update(status='waiting_budget'))
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],run['budget_request']['digest'],True)
self.assertEqual(err.exception.code,'MODEL_RESULT_UNKNOWN')
self.assertEqual(rt.store.calls(run['id'])[0]['reservation'],50)
self.assertEqual(model.count,0)
class CompilerEditTests(unittest.TestCase):
setUp=gateway_fixtures.BailianGatewayTests.setUp
provider=gateway_fixtures.BailianGatewayTests.provider
response=gateway_fixtures.BailianGatewayTests.response
generate=gateway_fixtures.BailianGatewayTests.generate
calls=gateway_fixtures.BailianGatewayTests.calls
def test_compiler_edit_applies_only_exact_source_and_retains_page(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business'))
files={'src/main/java/demo/App.java':'for(int i=0;i<2;i++){Inventory i = make(); rows.add(i);}',
'website/index.html':'<p>unchanged page</p>'}
data=ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',{'diagnostics':'/input/src/main/java/demo/App.java:1: error: variable i is already defined'})
edits={'edits':[{'old':'Inventory i = make(); rows.add(i);','new':'Inventory item = make(); rows.add(item);'}]}
result=self.generate([self.provider(lambda r:self.response(json.dumps(edits)))],kind='repair',payload=data)
self.assertIn('Inventory item',result['files']['src/main/java/demo/App.java'])
fake={**self.runtime.store.get(self.run['id']),'repair_pending':True,'outputs':{'patch':{'source':{'files':files}}},
'release':{'draft':{'graph':{'nodes':[{'id':'patch','operation_ref':'AO06@0.1.0'}]}}}}
merged=AgentRuntime._candidate_patch(fake,result)
self.assertEqual(merged['files']['website/index.html'],files['website/index.html'])
def test_unknown_cannot_be_replayed_or_reservation_released(self):
def timeout(request):raise httpx.ReadTimeout('secret must not be recorded')
with self.assertRaises(SkillError):self.generate([self.provider(timeout)])
before=self.calls()
with self.assertRaises(SkillError) as error:self.generate([self.provider(lambda r:self.response('{}'))])
self.assertEqual(error.exception.code,'MODEL_RESULT_UNKNOWN')
self.assertEqual(self.calls(),before)
self.assertEqual(before[0]['metadata']['reason_code'],'READ_TIMEOUT')
def test_oversize_is_saved_without_dispatch(self):
with self.assertRaises(SkillError) as error:self.generate([self.provider(lambda r:self.fail('must not dispatch'))],payload={'requirement':'x'*40000})
self.assertEqual(error.exception.code,'STAGE_CONTEXT_BUDGET_EXCEEDED')
self.assertEqual(self.calls(),[])
check=self.runtime.store.get(self.run['id'])['budget_checks'][-1]
self.assertGreater(check['parts']['requirement']['utf8_bytes'],40000)
def test_invalid_or_ambiguous_edits_do_not_change_saved_candidate(self):
data={'previous':{'src/main/java/demo/App.java':'x x'},'repair_scope':{'compiler_only':True}}
with self.assertRaises(SkillError):
self.generate([self.provider(lambda r:self.response(json.dumps({'edits':[{'old':'x','new':'y'}]})))],kind='repair',payload=data)
self.assertEqual(data['previous']['src/main/java/demo/App.java'],'x x')
def test_related_java_and_page_edits_use_full_saved_base_and_reject_unseen_text(self):
files={'src/main/java/demo/App.java':'class App { int limit=1000; }',
'website/index.html':'<style>must stay</style><script>const limit=1000;</script>'}
self.runtime.store.mutate(self.run['id'],lambda r:r.update(outputs={'patch':{'source':{'files':files}}}))
data={'previous':{'src/main/java/demo/App.java':files['src/main/java/demo/App.java'],
'website/index.html':'const limit=1000;'},'repair_scope':{'mode':'exact-edits'}}
edits={'edits':[{'path':path,'old':'limit=1000','new':'limit=50'} for path in files]}
result=self.generate([self.provider(lambda r:self.response(json.dumps(edits)))],kind='repair',payload=data)
self.assertIn('<style>must stay</style>',result['files']['website/index.html'])
self.assertTrue(all('limit=50' in text for text in result['files'].values()))
self.assertTrue(all('limit=1000' in text for text in files.values()))
def test_missing_edit_path_is_diagnosable_and_candidate_is_not_applied(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['budget'].update(max_model_calls=1))
data={'previous':{'src/main/java/demo/App.java':'int limit=1000;'},'repair_scope':{'mode':'exact-edits'}}
with self.assertRaises(SkillError):
self.generate([self.provider(lambda r:self.response(json.dumps({'edits':[{'old':'limit=1000','new':'limit=50'}]})))],kind='repair',payload=data)
call=self.calls()[0]
self.assertEqual(call['state'],'settled')
self.assertIn(['edits',0,'path'],[e['location'] for e in call['metadata']['validation_errors']])
self.assertEqual(data['previous']['src/main/java/demo/App.java'],'int limit=1000;')
if __name__=='__main__':unittest.main()