VibeCoding / src /tests /runtime /test_failure_localization.py
dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
75835df verified
Raw History Blame Contribute Delete
4.54 kB
"""Explicit synthetic diagnosis contracts; live source/VM tests are separate."""
import json
import unittest
from src.tests.runtime.fixtures import setup,workspace,USER,AUTHOR,SOURCE
from src.modules.skill.atomic_registry.catalog import routing_operations
from src.modules.skill.domain.models import Draft,Budget
from src.modules.llm_gateway.service import ModelReply,ProviderFailure
from src.modules.governance.local_validation import LocalValidation
class FailureLocalizationTests(unittest.TestCase):
def execute(self, mode='valid'):
runtime,request,model,sandbox=setup(self.enterContext(workspace()))
for op in routing_operations():runtime.skills.add_operation(AUTHOR,op)
doc=runtime.skills.read_draft(AUTHOR,'erp-java-development')
body={k:v for k,v in doc['body'].items() if k not in {'owner_id','last_editor_id'}}
body['project_id']='fixture-project'
body['budget']=Budget(max_model_calls=6,max_tokens=120000,max_repairs=2).model_dump()
for node in body['graph']['nodes']:
if node.get('operation_ref')=='AO05@0.1.0':node['operation_ref']='AO05@0.2.0'
runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),doc['revision'])
release=runtime.skills.stage_local_validation(AUTHOR,'erp-java-development','unit-localization',validation_owner=USER.user_id)
request['release_ref']=release['key']
model.task_roles=['diagnose'];captured=[];original=model.complete
sandbox.fail='build'
def complete(messages,*args):
data=json.loads(messages[-1]['content']);captured.append(data)
if data['task']=='diagnose':
model.count+=1
if mode=='unknown':raise ProviderFailure('MODEL_RESULT_UNKNOWN')
if mode=='invalid':return ModelReply({'files':SOURCE},80,80,model.name)
return ModelReply({'summary':'Concrete synthetic compiler cause and minimal correction.'},80,80,model.name)
if data['task']=='repair' and data['data'].get('failure_analysis'):sandbox.fail=None
return original(messages,*args)
model.complete=complete
run=runtime.create(USER,request,'failure-localization-unit')
validator=LocalValidation(runtime,USER,'unit-localization')
for _ in range(30):
run=runtime.get(USER,run['id'])
if run['status'] in {'unknown','failed'} or (run.get('pending_approval') or {}).get('scope')=='acceptance':break
if mode=='one-call-left' and run['repairs']==2:
runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_model_calls=4))
validator.step(run['id'])
return runtime,runtime.get(USER,run['id']),captured
def test_repeat_failure_is_localized_then_repaired_and_independently_retested(self):
runtime,run,messages=self.execute()
self.assertEqual(run['status'],'waiting_approval')
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertEqual([m['task'] for m in messages],['design','code','repair','diagnose','repair'])
self.assertEqual(run['failure_analysis']['trust'],'model_hypothesis_requires_independent_tests')
self.assertEqual(runtime.validation_evidence(USER,run['id'])['manifest']['failure_analysis'],run['failure_analysis'])
self.assertEqual(len(runtime.store.calls(run['id'])),5)
def test_unknown_diagnosis_stops_without_applying_or_replaying(self):
runtime,run,messages=self.execute('unknown')
self.assertEqual(run['status'],'unknown')
self.assertEqual(messages[-1]['task'],'diagnose')
self.assertEqual(runtime.store.calls(run['id'])[-1]['state'],'unknown')
def test_invalid_optional_diagnosis_is_settled_once_and_does_not_consume_a_retry(self):
runtime,run,messages=self.execute('invalid')
self.assertEqual(run['status'],'failed')
self.assertEqual(sum(m['task']=='diagnose' for m in messages),1)
self.assertEqual(len(runtime.store.calls(run['id'])),5)
self.assertTrue(all(c['state']=='settled' for c in runtime.store.calls(run['id'])))
def test_diagnosis_does_not_steal_the_last_repair_call_or_expand_budget(self):
runtime,run,messages=self.execute('one-call-left')
self.assertEqual(run['status'],'failed')
self.assertFalse(any(m['task']=='diagnose' for m in messages))
self.assertEqual(len(runtime.store.calls(run['id'])),4)