File size: 4,540 Bytes
75835df
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""Explicit synthetic diagnosis contracts; live source/VM tests are separate."""
import json
import unittest
from src.tests.runtime.fixtures import setup,workspace,USER,AUTHOR,SOURCE
from src.modules.skill.atomic_registry.catalog import routing_operations
from src.modules.skill.domain.models import Draft,Budget
from src.modules.llm_gateway.service import ModelReply,ProviderFailure
from src.modules.governance.local_validation import LocalValidation


class FailureLocalizationTests(unittest.TestCase):
    def execute(self, mode='valid'):
        runtime,request,model,sandbox=setup(self.enterContext(workspace()))
        for op in routing_operations():runtime.skills.add_operation(AUTHOR,op)
        doc=runtime.skills.read_draft(AUTHOR,'erp-java-development')
        body={k:v for k,v in doc['body'].items() if k not in {'owner_id','last_editor_id'}}
        body['project_id']='fixture-project'
        body['budget']=Budget(max_model_calls=6,max_tokens=120000,max_repairs=2).model_dump()
        for node in body['graph']['nodes']:
            if node.get('operation_ref')=='AO05@0.1.0':node['operation_ref']='AO05@0.2.0'
        runtime.skills.save_draft(AUTHOR,Draft.model_validate(body),doc['revision'])
        release=runtime.skills.stage_local_validation(AUTHOR,'erp-java-development','unit-localization',validation_owner=USER.user_id)
        request['release_ref']=release['key']
        model.task_roles=['diagnose'];captured=[];original=model.complete
        sandbox.fail='build'
        def complete(messages,*args):
            data=json.loads(messages[-1]['content']);captured.append(data)
            if data['task']=='diagnose':
                model.count+=1
                if mode=='unknown':raise ProviderFailure('MODEL_RESULT_UNKNOWN')
                if mode=='invalid':return ModelReply({'files':SOURCE},80,80,model.name)
                return ModelReply({'summary':'Concrete synthetic compiler cause and minimal correction.'},80,80,model.name)
            if data['task']=='repair' and data['data'].get('failure_analysis'):sandbox.fail=None
            return original(messages,*args)
        model.complete=complete
        run=runtime.create(USER,request,'failure-localization-unit')
        validator=LocalValidation(runtime,USER,'unit-localization')
        for _ in range(30):
            run=runtime.get(USER,run['id'])
            if run['status'] in {'unknown','failed'} or (run.get('pending_approval') or {}).get('scope')=='acceptance':break
            if mode=='one-call-left' and run['repairs']==2:
                runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_model_calls=4))
            validator.step(run['id'])
        return runtime,runtime.get(USER,run['id']),captured

    def test_repeat_failure_is_localized_then_repaired_and_independently_retested(self):
        runtime,run,messages=self.execute()
        self.assertEqual(run['status'],'waiting_approval')
        self.assertEqual(run['pending_approval']['scope'],'acceptance')
        self.assertEqual([m['task'] for m in messages],['design','code','repair','diagnose','repair'])
        self.assertEqual(run['failure_analysis']['trust'],'model_hypothesis_requires_independent_tests')
        self.assertEqual(runtime.validation_evidence(USER,run['id'])['manifest']['failure_analysis'],run['failure_analysis'])
        self.assertEqual(len(runtime.store.calls(run['id'])),5)

    def test_unknown_diagnosis_stops_without_applying_or_replaying(self):
        runtime,run,messages=self.execute('unknown')
        self.assertEqual(run['status'],'unknown')
        self.assertEqual(messages[-1]['task'],'diagnose')
        self.assertEqual(runtime.store.calls(run['id'])[-1]['state'],'unknown')

    def test_invalid_optional_diagnosis_is_settled_once_and_does_not_consume_a_retry(self):
        runtime,run,messages=self.execute('invalid')
        self.assertEqual(run['status'],'failed')
        self.assertEqual(sum(m['task']=='diagnose' for m in messages),1)
        self.assertEqual(len(runtime.store.calls(run['id'])),5)
        self.assertTrue(all(c['state']=='settled' for c in runtime.store.calls(run['id'])))

    def test_diagnosis_does_not_steal_the_last_repair_call_or_expand_budget(self):
        runtime,run,messages=self.execute('one-call-left')
        self.assertEqual(run['status'],'failed')
        self.assertFalse(any(m['task']=='diagnose' for m in messages))
        self.assertEqual(len(runtime.store.calls(run['id'])),4)