Spaces:
Sleeping
Sleeping
File size: 10,895 Bytes
8f4045a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 | """Protocol tests for review provenance, budgets and fail-closed execution."""
import json
import unittest
from src.tests.runtime.fixtures import setup, workspace, AUTHOR, REVIEWER, USER, SOURCE, through_execution
from src.modules.skill.atomic_registry.catalog import code_review_operation
from src.modules.skill.packs.code_review import with_code_review, without_code_review
from src.modules.llm_gateway.service import ModelReply, ProviderFailure
from src.modules.skill.domain.models import digest
class CodeReviewTests(unittest.TestCase):
def setUp(self):
self.runtime,self.request,self.model,self.sandbox=setup(self.enterContext(workspace()))
skills=self.runtime.skills
skills.add_operation(AUTHOR,code_review_operation())
old=skills.read_draft(AUTHOR,'erp-java-development')
body={k:v for k,v in old['body'].items() if k not in {'owner_id','last_editor_id'}}
body['budget']={'max_model_calls':6,'max_tokens':120000,'max_repairs':2,'max_seconds':1800}
self.draft=with_code_review(body)
skills.save_draft(AUTHOR,self.draft,old['revision'])
report=skills.validate(AUTHOR,self.draft.skill_id)
self.assertTrue(report.valid,report.issues)
approval=skills.approve(REVIEWER,self.draft.skill_id,old['revision']+1,report.digest,['unit-protocol-only'])
release=skills.publish(AUTHOR,self.draft.skill_id,'0.2.0',old['revision']+1,approval['key'])
self.request['release_ref']=release['key']
self.tasks=[];self.review_mode='normal'
def complete(messages,max_tokens,cancelled):
task=json.loads(messages[-1]['content'])['task'];self.tasks.append(task)
files={p:text.replace('public class','public final class') for p,text in SOURCE.items()}
if task=='code_review':
if self.review_mode=='unknown':raise ProviderFailure()
if self.review_mode=='unsafe':files={'../escape.java':'bad'}
if self.review_mode=='extra':files={'website/index.html':'<html></html>'}
if self.review_mode=='partial':files={'website/index.html':'<html></html>',
'src/main/java/demo/App.java':SOURCE['src/main/java/demo/App.java'].replace('public class','public final class')}
content={'summary':'检视候选','findings':['App.java: make utility class final'],'files':files}
elif task=='design':content={'summary':'Synthetic design'}
elif task=='repair':content={'files':{p:text.replace('public class','final class') for p,text in SOURCE.items()}}
else:content={'files':SOURCE}
return ModelReply(content,80,80,self.model.name)
self.model.complete=complete
def test_builtin_review_removal_runs_directly_to_sandbox(self):
skills=self.runtime.skills
current=skills.read_draft(AUTHOR,self.draft.skill_id)
body={k:v for k,v in current['body'].items() if k not in {'owner_id','last_editor_id'}}
draft=without_code_review(body)
skills.save_draft(AUTHOR,draft,current['revision'])
report=skills.validate(AUTHOR,draft.skill_id)
self.assertTrue(report.valid,report.issues)
approval=skills.approve(REVIEWER,draft.skill_id,current['revision']+1,report.digest,['unit-protocol-only'])
release=skills.publish(AUTHOR,draft.skill_id,'0.3.0',current['revision']+1,approval['key'])
self.request['release_ref']=release['key']
run=through_execution(self.runtime,self.request)
self.assertEqual(self.tasks,['design','code'])
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
self.assertNotIn('code_review',run['outputs'])
self.assertEqual(run['outputs']['patch']['source']['files'],SOURCE)
def test_review_changes_candidate_before_build_and_retains_original(self):
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertEqual(self.tasks,['design','code','code_review'])
record=run['outputs']['code_review']['review']
self.assertEqual(record['original_candidate'],SOURCE)
self.assertEqual(record['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
self.assertEqual(record['trust'],'model_review_requires_independent_tests')
self.assertEqual(record['changed_paths'],list(SOURCE))
evidence=self.runtime.validation_evidence(USER,run['id'])
self.assertEqual(evidence['manifest']['development']['code_review'],record)
self.assertEqual(run['budget']['max_model_calls'],6)
def test_unknown_review_is_not_replayed_or_built(self):
self.review_mode='unknown'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['status'],'unknown')
self.assertEqual(self.tasks.count('code_review'),1)
self.assertNotIn('build',self.sandbox.calls)
self.assertEqual(self.runtime.store.calls(run['id'])[-1]['state'],'unknown')
def test_java_only_continuation_review_sees_the_actual_unchanged_page(self):
from unittest.mock import patch
from src.modules.execution.scenarios import business_app
run=self.runtime.create(USER,self.request,'review-baseline-001')
framework=run['snapshot']['framework']
framework.update(profile_id=business_app.PROFILE,allowed_paths=['src/main/','website/'])
page='<html><style>body{color:black}</style><body>Existing page<script>const field="original";</script></body></html>'
files={**SOURCE,business_app.PAGE:page}
run['request'].update(baseline={'files':files,'source_digest':digest(files)},recovery_diagnostics={'observed':{}})
observed={}
def review(run_id,kind,payload,*args,**kwargs):
observed.update(payload)
return {'summary':'检视真实页面契约','findings':[],'files':payload['patch']['files']}
with patch.object(self.runtime.llm,'generate',side_effect=review):
result=self.runtime._handle(USER,run,code_review_operation().model_dump(),
{'patch':{'files':SOURCE},'design':{'summary':'Continue existing business'}},lambda:False)
self.assertEqual(observed['patch']['files'][business_app.PAGE],page)
self.assertEqual(result['patch']['files'],files)
def test_review_output_never_expands_paths_and_is_sanitized(self):
# A review model adding a benign-looking path outside the candidate must not
# fail the run: unknown paths are dropped and recorded, candidate is rebuilt.
self.review_mode='extra'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['website/index.html'])
self.assertEqual(record['reviewed_candidate'],SOURCE)
self.assertEqual(record['changed_paths'],[])
def test_review_omitted_candidate_paths_are_refilled_from_original(self):
self.review_mode='partial'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['website/index.html'])
self.assertEqual(record['reviewed_candidate'],
{p:t.replace('public class','public final class') for p,t in SOURCE.items()})
self.assertEqual(record['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
def test_reviewed_patch_still_passes_harness(self):
# Escape-style paths are also sanitized (never built), with the drop on record;
# the harness re-validates the merged patch before build regardless.
self.review_mode='unsafe'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['../escape.java'])
self.assertEqual(run['outputs']['patch']['source']['files'],SOURCE)
def test_exhausted_shared_budget_blocks_review_without_skipping_to_build(self):
run=self.runtime.create(USER,self.request,'review-budget-001')
self.runtime.advance(USER,run['id'])
run=self.runtime.get(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
run=self.runtime.advance(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
self.runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_tokens=350))
run=self.runtime.advance(USER,run['id'])
self.assertEqual(run['status'],'waiting_budget')
self.assertEqual(run['error'],'TASK_TOKEN_BUDGET_EXCEEDED')
self.assertNotIn('code_review',self.tasks)
self.assertNotIn('build',self.sandbox.calls)
def test_later_repair_uses_new_patch_without_claiming_it_was_reviewed(self):
self.sandbox.fail='build'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['status'],'repair_required')
self.sandbox.fail=None
self.runtime.repair(USER,run['id'])
run=self.runtime.advance(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
run=self.runtime.advance(USER,run['id'])
review=run['outputs']['code_review']['review']
self.assertTrue(review['superseded_by_repair'])
self.assertNotEqual(review['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
self.assertEqual(self.tasks.count('code_review'),1)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
def test_graph_rejects_bypass_of_review_and_preserves_old_release(self):
graph=self.draft.model_dump()
for edge in graph['graph']['data_edges']:
if edge['target']=='patch':edge['source']='code'
from src.modules.skill.domain.models import Draft
current=self.runtime.skills.read_draft(AUTHOR,self.draft.skill_id)
self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(graph),current['revision'])
report=self.runtime.skills.validate(AUTHOR,self.draft.skill_id)
self.assertFalse(report.valid)
self.assertTrue(any(i.code=='UNTRUSTED_STAGE_INPUT' for i in report.issues))
|