Spaces:
Sleeping
Sleeping
File size: 17,098 Bytes
8f4045a 115b94d 8f4045a 115b94d 8f4045a 115b94d 8f4045a 115b94d 8f4045a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 | """Budget boundaries and compiler edits; synthetic protocol, no paid calls."""
import json
import unittest
import httpx
from types import SimpleNamespace
from src.tests.runtime import test_bailian_gateway as gateway_fixtures
from src.modules.llm_gateway.service import LLMGateway,ProviderFailure
from src.modules.context.service import ContextService
from src.modules.orchestrator.service import AgentRuntime
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup,workspace,USER,OTHER
class PreflightTests(unittest.TestCase):
def check(self,stage=32000,total=120000,calls=(),window=None,text='x'):
return LLMGateway.preflight([{'content':'system'},{'content':text}],{'requirement':text},
{'budget':{'max_tokens':total}},calls,stage,12288,SimpleNamespace(context_window=window),'repair')[1]
def test_normal_keeps_safety_and_output_reservation(self):
c=self.check();self.assertIsNone(c['rejection'])
self.assertEqual(c['input_bound'],519)
self.assertEqual(c['output_reservation'],12288)
self.assertFalse(c['model_context_limit_verified'])
def test_distinct_stage_task_and_model_rejections(self):
for options,code in [({'stage':600},'STAGE_CONTEXT_BUDGET_EXCEEDED'),
({'total':600},'TASK_TOKEN_BUDGET_EXCEEDED'),
({'window':600},'MODEL_CONTEXT_LIMIT_EXCEEDED')]:
with self.subTest(code=code):self.assertEqual(self.check(**options)['rejection'][0],code)
def test_exact_output_floor_and_unknown_reservations_count(self):
self.assertIsNone(self.check(stage=647)['rejection'])
self.assertEqual(self.check(stage=646)['rejection'][0],'STAGE_CONTEXT_BUDGET_EXCEEDED')
c=self.check(total=1000,calls=[{'charged':None,'reservation':400}])
self.assertEqual(c['reserved_tokens'],400)
self.assertEqual(c['task_remaining'],600)
self.assertEqual(c['rejection'][0],'TASK_TOKEN_BUDGET_EXCEEDED')
def test_compiler_scope_keeps_full_java_and_hashes_unchanged_page(self):
files={'src/main/java/demo/App.java':'complete Java','website/index.html':'complete page'}
diag={'diagnostics':'/input/src/main/java/demo/App.java:268: error: duplicate variable\ncode\n ^'}
inputs=ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',diag)
self.assertEqual(inputs['previous'],{'src/main/java/demo/App.java':'complete Java'})
self.assertIn('website/index.html',inputs['repair_scope']['unchanged_files'])
for code in ['JAVA_TEST_FAILED','MODEL_OUTPUT_INVALID']:
self.assertEqual(ContextService.repair_inputs(files,code,diag)['previous'],files)
self.assertEqual(ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',{})['previous'],files)
def test_list_failure_keeps_both_logic_views_and_explicit_omission_records(self):
files={'src/main/java/demo/App.java':' static void generateInitialData() { /* unrelated */ }\n static void handleList() { /* relevant */ }\n',
'website/index.html':'<style>retained</style><button id="next">Next</button><script>function load(){return "records";}</script>'}
data=ContextService.repair_inputs(files,'JAVA_TEST_FAILED',{'failures':[{'path':'/api/app/list'}]})
self.assertEqual(data['repair_scope']['mode'],'exact-edits')
self.assertIn('handleList',data['previous']['src/main/java/demo/App.java'])
self.assertIn('load()',data['previous']['website/index.html'])
self.assertEqual(data['repair_scope']['omitted_compiled_methods'][0]['method'],'generateInitialData')
self.assertIn('id="next"',data['page_elements'][0])
self.assertIn('<style>retained</style>',files['website/index.html'])
class BudgetConsentTests(unittest.TestCase):
def test_repair_resume_retains_original_failure_and_identical_context(self):
from src.modules.llm_gateway.service import ModelReply
with workspace() as folder:
rt,request,model,_=setup(folder)
request={**request,'requirement':'帮我开发一个物流管理的程序,作为物流公司,能够检索到当前订单的状态,并根据实际的物流进行状态调整。'}
run=rt.create(USER,request,'logistics-repair-resume')
java='src/main/java/demo/App.java'
files={java:'package demo; public class App { int duplicate=0; int duplicate=1; public static void main(String[]a){} } /*'+('x'*24000)+'*/'}
def prepare(r):
nodes=r['release']['draft']['graph']['nodes']
node=lambda op:next(n['id'] for n in nodes if (n.get('operation_ref') or '').startswith(op+'@'))
r.update(repair_pending=True,repairs=1,error='JAVA_BUILD_FAILED',
diagnostics={'diagnostics':'/input/src/main/java/demo/App.java:1: error: variable duplicate is already defined'},
outputs={node('AO04'):{'design':{'summary':'固定原始物流需求,不扩展模块。'}},node('AO06'):{'source':{'files':files}}})
r['budget']['max_tokens']=120000
rt.store.mutate(run['id'],prepare)
def complete(messages,bound,cancelled):
payload=json.loads(messages[-1]['content'])['data']
self.assertEqual(payload['failure_code'],'JAVA_BUILD_FAILED')
self.assertTrue(payload['repair_scope']['compiler_only'])
model.count+=1
return ModelReply({'edits':[{'old':'int duplicate=0; int duplicate=1;','new':'int duplicate=0; int fixed=1;'}]},100,20,model.name)
model.complete=complete
paused=rt.advance(USER,run['id']);self.assertEqual(paused['status'],'waiting_budget')
p=paused['budget_request'];self.assertEqual(paused['repair_failure_code'],'JAVA_BUILD_FAILED')
rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
resumed=rt.advance(USER,run['id'])
self.assertEqual(resumed['status'],'waiting_approval');self.assertEqual(model.count,1)
self.assertEqual(len(rt.store.calls(run['id'])),1)
self.assertEqual(rt.store.calls(run['id'])[0]['metadata']['budget_check']['input_bound'],p['input_bound'])
def paused(self,folder):
runtime,request,model,_=setup(folder)
run=runtime.create(USER,request,'budget-consent-001')
runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_tokens=100))
run=runtime.advance(USER,run['id'])
self.assertEqual(run['status'],'waiting_budget')
self.assertEqual(model.count,0)
self.assertEqual(runtime.store.calls(run['id']),[])
return runtime,run,model
def test_explicit_approval_resumes_same_stage_and_preserves_snapshot(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
pending=run['budget_request'];snapshot=run['snapshot']
with self.assertRaises(SkillError):rt.decide_budget(OTHER,run['id'],pending['digest'],True)
with self.assertRaises(SkillError):rt.decide_budget(USER,run['id'],'0'*64,True)
approved=rt.decide_budget(USER,run['id'],pending['digest'],True)
self.assertEqual(approved['id'],run['id'])
self.assertEqual(approved['snapshot'],snapshot)
self.assertEqual(approved['budget']['max_tokens'],pending['task_limit'])
duplicate=rt.decide_budget(USER,run['id'],pending['digest'],True)
self.assertFalse(duplicate['budget_decision_applied'])
self.assertEqual(duplicate['revision'],approved['revision'])
resumed=rt.advance(USER,run['id'])
self.assertEqual(resumed['status'],'waiting_approval')
self.assertEqual(model.count,1)
self.assertEqual(len(resumed['budget_authorizations']),1)
def test_declining_preserves_evidence_and_never_calls_model(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
result=rt.decide_budget(USER,run['id'],run['budget_request']['digest'],False)
self.assertEqual(result['status'],'waiting_budget')
self.assertTrue(result['budget_request']['deferred'])
self.assertEqual(result['budget']['max_tokens'],100)
self.assertTrue(result['budget_checks'])
self.assertEqual(model.count,0)
def test_expiry_recalculates_and_requires_a_new_confirmation(self):
with workspace() as folder:
rt,run,model=self.paused(folder);pending=run['budget_request']
rt.clock=lambda:pending['expires_at']+1
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],pending['digest'],True,pending['task_revision'])
self.assertEqual(err.exception.code,'BUDGET_REQUEST_CHANGED')
fresh=rt.store.get(run['id'])
self.assertNotEqual(fresh['budget_request']['digest'],pending['digest'])
self.assertEqual(fresh['budget']['max_tokens'],100);self.assertEqual(model.count,0)
rt.decide_budget(USER,run['id'],fresh['budget_request']['digest'],True,fresh['budget_request']['task_revision'])
self.assertEqual(model.count,0) # Decision enqueues only; no direct dispatch.
def test_changed_task_version_or_context_never_uses_stale_approval(self):
for context_changed in (False,True):
with self.subTest(context_changed=context_changed),workspace() as folder:
rt,run,model=self.paused(folder);p=run['budget_request']
rt.store.mutate(run['id'],lambda r:r['outputs'].update(changed=True) if context_changed else r.update(audit_marker=True))
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
self.assertEqual(err.exception.code,'BUDGET_REQUEST_CHANGED')
changed=rt.store.get(run['id']);self.assertEqual(changed['budget']['max_tokens'],100)
self.assertEqual(model.count,0)
self.assertEqual(changed['status'],'failed' if context_changed else 'waiting_budget')
def test_model_window_cannot_be_overridden_by_approval(self):
with workspace() as folder:
rt,run,model=self.paused(folder);p=run['budget_request']
model.context_window=1
with self.assertRaises(SkillError):rt.decide_budget(USER,run['id'],p['digest'],True,p['task_revision'])
refused=rt.store.get(run['id']);self.assertEqual(refused['error'],'MODEL_CONTEXT_LIMIT_EXCEEDED')
self.assertIsNone(refused['budget_request']);self.assertEqual(model.count,0)
def test_environment_and_supplier_refusals_never_create_budget_approval(self):
for code in ('SANDBOX_BOOT_FAILED','JAVA_TEST_FAILED','MODEL_QUOTA_REJECTED'):
with self.subTest(code=code),workspace() as folder:
rt,request,model,_=setup(folder);run=rt.create(USER,request,'not-budget-'+code)
def fail(*a,**kw):raise SkillError(code,'synthetic fault',409)
rt.llm.generate=fail;out=rt.advance(USER,run['id'])
self.assertNotEqual(out['status'],'waiting_budget');self.assertIsNone(out.get('budget_request'))
self.assertEqual(model.count,0)
def test_approval_cannot_release_unknown_reservation(self):
with workspace() as folder:
rt,run,model=self.paused(folder)
rt.store.mutate(run['id'],lambda r:r.update(status='running'))
call=rt.store.reserve(run['id'],'synthetic',[],50)
rt.store.settle(call,'unknown')
rt.store.mutate(run['id'],lambda r:r.update(status='waiting_budget'))
with self.assertRaises(SkillError) as err:rt.decide_budget(USER,run['id'],run['budget_request']['digest'],True)
self.assertEqual(err.exception.code,'MODEL_RESULT_UNKNOWN')
self.assertEqual(rt.store.calls(run['id'])[0]['reservation'],50)
self.assertEqual(model.count,0)
class CompilerEditTests(unittest.TestCase):
setUp=gateway_fixtures.BailianGatewayTests.setUp
provider=gateway_fixtures.BailianGatewayTests.provider
response=gateway_fixtures.BailianGatewayTests.response
generate=gateway_fixtures.BailianGatewayTests.generate
calls=gateway_fixtures.BailianGatewayTests.calls
def test_compiler_edit_applies_only_exact_source_and_retains_page(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['snapshot']['framework'].update(profile_id='java17-requirement-business'))
files={'src/main/java/demo/App.java':'for(int i=0;i<2;i++){Inventory i = make(); rows.add(i);}',
'website/index.html':'<p>unchanged page</p>'}
data=ContextService.repair_inputs(files,'JAVA_BUILD_FAILED',{'diagnostics':'/input/src/main/java/demo/App.java:1: error: variable i is already defined'})
edits={'edits':[{'old':'Inventory i = make(); rows.add(i);','new':'Inventory item = make(); rows.add(item);'}]}
result=self.generate([self.provider(lambda r:self.response(json.dumps(edits)))],kind='repair',payload=data)
self.assertIn('Inventory item',result['files']['src/main/java/demo/App.java'])
fake={**self.runtime.store.get(self.run['id']),'repair_pending':True,'outputs':{'patch':{'source':{'files':files}}},
'release':{'draft':{'graph':{'nodes':[{'id':'patch','operation_ref':'AO06@0.1.0'}]}}}}
merged=AgentRuntime._candidate_patch(fake,result)
self.assertEqual(merged['files']['website/index.html'],files['website/index.html'])
def test_unknown_cannot_be_replayed_or_reservation_released(self):
def timeout(request):raise httpx.ReadTimeout('secret must not be recorded')
with self.assertRaises(SkillError):self.generate([self.provider(timeout)])
before=self.calls()
with self.assertRaises(SkillError) as error:self.generate([self.provider(lambda r:self.response('{}'))])
self.assertEqual(error.exception.code,'MODEL_RESULT_UNKNOWN')
self.assertEqual(self.calls(),before)
self.assertEqual(before[0]['metadata']['reason_code'],'READ_TIMEOUT')
def test_oversize_is_saved_without_dispatch(self):
with self.assertRaises(SkillError) as error:self.generate([self.provider(lambda r:self.fail('must not dispatch'))],payload={'requirement':'x'*40000})
self.assertEqual(error.exception.code,'STAGE_CONTEXT_BUDGET_EXCEEDED')
self.assertEqual(self.calls(),[])
check=self.runtime.store.get(self.run['id'])['budget_checks'][-1]
self.assertGreater(check['parts']['requirement']['utf8_bytes'],40000)
def test_invalid_or_ambiguous_edits_do_not_change_saved_candidate(self):
data={'previous':{'src/main/java/demo/App.java':'x x'},'repair_scope':{'compiler_only':True}}
with self.assertRaises(SkillError):
self.generate([self.provider(lambda r:self.response(json.dumps({'edits':[{'old':'x','new':'y'}]})))],kind='repair',payload=data)
self.assertEqual(data['previous']['src/main/java/demo/App.java'],'x x')
def test_related_java_and_page_edits_use_full_saved_base_and_reject_unseen_text(self):
files={'src/main/java/demo/App.java':'class App { int limit=1000; }',
'website/index.html':'<style>must stay</style><script>const limit=1000;</script>'}
self.runtime.store.mutate(self.run['id'],lambda r:r.update(outputs={'patch':{'source':{'files':files}}}))
data={'previous':{'src/main/java/demo/App.java':files['src/main/java/demo/App.java'],
'website/index.html':'const limit=1000;'},'repair_scope':{'mode':'exact-edits'}}
edits={'edits':[{'path':path,'old':'limit=1000','new':'limit=50'} for path in files]}
result=self.generate([self.provider(lambda r:self.response(json.dumps(edits)))],kind='repair',payload=data)
self.assertIn('<style>must stay</style>',result['files']['website/index.html'])
self.assertTrue(all('limit=50' in text for text in result['files'].values()))
self.assertTrue(all('limit=1000' in text for text in files.values()))
def test_missing_edit_path_is_diagnosable_and_candidate_is_not_applied(self):
self.runtime.store.mutate(self.run['id'],lambda r:r['budget'].update(max_model_calls=1))
data={'previous':{'src/main/java/demo/App.java':'int limit=1000;'},'repair_scope':{'mode':'exact-edits'}}
with self.assertRaises(SkillError):
self.generate([self.provider(lambda r:self.response(json.dumps({'edits':[{'old':'limit=1000','new':'limit=50'}]})))],kind='repair',payload=data)
call=self.calls()[0]
self.assertEqual(call['state'],'settled')
self.assertIn(['edits',0,'path'],[e['location'] for e in call['metadata']['validation_errors']])
self.assertEqual(data['previous']['src/main/java/demo/App.java'],'int limit=1000;')
if __name__=='__main__':unittest.main()
|