VibeCoding / src /tests /runtime /test_code_review.py
dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
8f4045a verified
Raw History Blame Contribute Delete
10.9 kB
"""Protocol tests for review provenance, budgets and fail-closed execution."""
import json
import unittest
from src.tests.runtime.fixtures import setup, workspace, AUTHOR, REVIEWER, USER, SOURCE, through_execution
from src.modules.skill.atomic_registry.catalog import code_review_operation
from src.modules.skill.packs.code_review import with_code_review, without_code_review
from src.modules.llm_gateway.service import ModelReply, ProviderFailure
from src.modules.skill.domain.models import digest
class CodeReviewTests(unittest.TestCase):
def setUp(self):
self.runtime,self.request,self.model,self.sandbox=setup(self.enterContext(workspace()))
skills=self.runtime.skills
skills.add_operation(AUTHOR,code_review_operation())
old=skills.read_draft(AUTHOR,'erp-java-development')
body={k:v for k,v in old['body'].items() if k not in {'owner_id','last_editor_id'}}
body['budget']={'max_model_calls':6,'max_tokens':120000,'max_repairs':2,'max_seconds':1800}
self.draft=with_code_review(body)
skills.save_draft(AUTHOR,self.draft,old['revision'])
report=skills.validate(AUTHOR,self.draft.skill_id)
self.assertTrue(report.valid,report.issues)
approval=skills.approve(REVIEWER,self.draft.skill_id,old['revision']+1,report.digest,['unit-protocol-only'])
release=skills.publish(AUTHOR,self.draft.skill_id,'0.2.0',old['revision']+1,approval['key'])
self.request['release_ref']=release['key']
self.tasks=[];self.review_mode='normal'
def complete(messages,max_tokens,cancelled):
task=json.loads(messages[-1]['content'])['task'];self.tasks.append(task)
files={p:text.replace('public class','public final class') for p,text in SOURCE.items()}
if task=='code_review':
if self.review_mode=='unknown':raise ProviderFailure()
if self.review_mode=='unsafe':files={'../escape.java':'bad'}
if self.review_mode=='extra':files={'website/index.html':'<html></html>'}
if self.review_mode=='partial':files={'website/index.html':'<html></html>',
'src/main/java/demo/App.java':SOURCE['src/main/java/demo/App.java'].replace('public class','public final class')}
content={'summary':'检视候选','findings':['App.java: make utility class final'],'files':files}
elif task=='design':content={'summary':'Synthetic design'}
elif task=='repair':content={'files':{p:text.replace('public class','final class') for p,text in SOURCE.items()}}
else:content={'files':SOURCE}
return ModelReply(content,80,80,self.model.name)
self.model.complete=complete
def test_builtin_review_removal_runs_directly_to_sandbox(self):
skills=self.runtime.skills
current=skills.read_draft(AUTHOR,self.draft.skill_id)
body={k:v for k,v in current['body'].items() if k not in {'owner_id','last_editor_id'}}
draft=without_code_review(body)
skills.save_draft(AUTHOR,draft,current['revision'])
report=skills.validate(AUTHOR,draft.skill_id)
self.assertTrue(report.valid,report.issues)
approval=skills.approve(REVIEWER,draft.skill_id,current['revision']+1,report.digest,['unit-protocol-only'])
release=skills.publish(AUTHOR,draft.skill_id,'0.3.0',current['revision']+1,approval['key'])
self.request['release_ref']=release['key']
run=through_execution(self.runtime,self.request)
self.assertEqual(self.tasks,['design','code'])
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
self.assertNotIn('code_review',run['outputs'])
self.assertEqual(run['outputs']['patch']['source']['files'],SOURCE)
def test_review_changes_candidate_before_build_and_retains_original(self):
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertEqual(self.tasks,['design','code','code_review'])
record=run['outputs']['code_review']['review']
self.assertEqual(record['original_candidate'],SOURCE)
self.assertEqual(record['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
self.assertEqual(record['trust'],'model_review_requires_independent_tests')
self.assertEqual(record['changed_paths'],list(SOURCE))
evidence=self.runtime.validation_evidence(USER,run['id'])
self.assertEqual(evidence['manifest']['development']['code_review'],record)
self.assertEqual(run['budget']['max_model_calls'],6)
def test_unknown_review_is_not_replayed_or_built(self):
self.review_mode='unknown'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['status'],'unknown')
self.assertEqual(self.tasks.count('code_review'),1)
self.assertNotIn('build',self.sandbox.calls)
self.assertEqual(self.runtime.store.calls(run['id'])[-1]['state'],'unknown')
def test_java_only_continuation_review_sees_the_actual_unchanged_page(self):
from unittest.mock import patch
from src.modules.execution.scenarios import business_app
run=self.runtime.create(USER,self.request,'review-baseline-001')
framework=run['snapshot']['framework']
framework.update(profile_id=business_app.PROFILE,allowed_paths=['src/main/','website/'])
page='<html><style>body{color:black}</style><body>Existing page<script>const field="original";</script></body></html>'
files={**SOURCE,business_app.PAGE:page}
run['request'].update(baseline={'files':files,'source_digest':digest(files)},recovery_diagnostics={'observed':{}})
observed={}
def review(run_id,kind,payload,*args,**kwargs):
observed.update(payload)
return {'summary':'检视真实页面契约','findings':[],'files':payload['patch']['files']}
with patch.object(self.runtime.llm,'generate',side_effect=review):
result=self.runtime._handle(USER,run,code_review_operation().model_dump(),
{'patch':{'files':SOURCE},'design':{'summary':'Continue existing business'}},lambda:False)
self.assertEqual(observed['patch']['files'][business_app.PAGE],page)
self.assertEqual(result['patch']['files'],files)
def test_review_output_never_expands_paths_and_is_sanitized(self):
# A review model adding a benign-looking path outside the candidate must not
# fail the run: unknown paths are dropped and recorded, candidate is rebuilt.
self.review_mode='extra'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['website/index.html'])
self.assertEqual(record['reviewed_candidate'],SOURCE)
self.assertEqual(record['changed_paths'],[])
def test_review_omitted_candidate_paths_are_refilled_from_original(self):
self.review_mode='partial'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['website/index.html'])
self.assertEqual(record['reviewed_candidate'],
{p:t.replace('public class','public final class') for p,t in SOURCE.items()})
self.assertEqual(record['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
def test_reviewed_patch_still_passes_harness(self):
# Escape-style paths are also sanitized (never built), with the drop on record;
# the harness re-validates the merged patch before build regardless.
self.review_mode='unsafe'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
self.assertIn('build',self.sandbox.calls)
record=run['outputs']['code_review']['review']
self.assertEqual(record['dropped_unknown_paths'],['../escape.java'])
self.assertEqual(run['outputs']['patch']['source']['files'],SOURCE)
def test_exhausted_shared_budget_blocks_review_without_skipping_to_build(self):
run=self.runtime.create(USER,self.request,'review-budget-001')
self.runtime.advance(USER,run['id'])
run=self.runtime.get(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
run=self.runtime.advance(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
self.runtime.store.mutate(run['id'],lambda r:r['budget'].update(max_tokens=350))
run=self.runtime.advance(USER,run['id'])
self.assertEqual(run['status'],'waiting_budget')
self.assertEqual(run['error'],'TASK_TOKEN_BUDGET_EXCEEDED')
self.assertNotIn('code_review',self.tasks)
self.assertNotIn('build',self.sandbox.calls)
def test_later_repair_uses_new_patch_without_claiming_it_was_reviewed(self):
self.sandbox.fail='build'
run=through_execution(self.runtime,self.request)
self.assertEqual(run['status'],'repair_required')
self.sandbox.fail=None
self.runtime.repair(USER,run['id'])
run=self.runtime.advance(USER,run['id'])
self.runtime.approve(USER,run['id'],run['pending_approval']['digest'])
run=self.runtime.advance(USER,run['id'])
review=run['outputs']['code_review']['review']
self.assertTrue(review['superseded_by_repair'])
self.assertNotEqual(review['reviewed_source_digest'],run['outputs']['patch']['source']['source_digest'])
self.assertEqual(self.tasks.count('code_review'),1)
self.assertEqual(run['pending_approval']['scope'],'acceptance')
def test_graph_rejects_bypass_of_review_and_preserves_old_release(self):
graph=self.draft.model_dump()
for edge in graph['graph']['data_edges']:
if edge['target']=='patch':edge['source']='code'
from src.modules.skill.domain.models import Draft
current=self.runtime.skills.read_draft(AUTHOR,self.draft.skill_id)
self.runtime.skills.save_draft(AUTHOR,Draft.model_validate(graph),current['revision'])
report=self.runtime.skills.validate(AUTHOR,self.draft.skill_id)
self.assertFalse(report.valid)
self.assertTrue(any(i.code=='UNTRUSTED_STAGE_INPUT' for i in report.issues))