VibeCoding / src /tests /runtime /test_qemu_contract.py
dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
5ca3c16 verified
Raw History Blame Contribute Delete
11.2 kB
import gzip
import hashlib
import json
import stat
import unittest
from pathlib import Path
from src.adapters.sandbox.qemu import QemuSandbox, VM_LAUNCH_V8
from src.modules.execution.contracts import SandboxConfig
from src.modules.skill.domain.models import SkillError, digest
from src.tests.runtime.fixtures import workspace, SOURCE
def unpack_overlay(raw):
entries = {}
offset = 0
while offset < len(raw):
assert raw[offset:offset + 6] == b'070701'
fields = [int(raw[offset + 6 + i * 8:offset + 14 + i * 8], 16) for i in range(13)]
offset += 110
name = raw[offset:offset + fields[11] - 1].decode()
offset = (offset + fields[11] + 3) & ~3
entries[name] = (fields[1], raw[offset:offset + fields[6]])
offset = (offset + fields[6] + 3) & ~3
if name == 'TRAILER!!!':
break
return entries
class QemuContractTests(unittest.TestCase):
def latest(self, command):
original=(self.image/'manifest.json').read_bytes()
manifest=json.loads(original)
control=Path(__file__).resolve().parents[2]/'modules/execution/container'
manifest.update(worker=hashlib.sha256((control/'vm_worker_v6.py').read_bytes()).hexdigest(),
probe=hashlib.sha256((control/'probe_v3.py').read_bytes()).hexdigest(),launch=VM_LAUNCH_V8)
(self.image/'manifest-v8.json').write_text(json.dumps(manifest))
config=self.sandbox.config.model_copy(update={'image':'zkcoding-qemu@sha256:'+digest(manifest)})
sandbox=QemuSandbox(self.root/'v8',config,executable=self.sandbox.executable,image_root=self.image,command=command)
sandbox.preflight();self.sandbox.preflight()
self.assertEqual((self.image/'manifest.json').read_bytes(),original)
return sandbox
def test_pre_worker_kernel_panic_restarts_same_input_once(self):
original=self.sandbox.command;attempts=[]
def command(argv,**options):
attempts.append((list(argv),Path(argv[argv.index('-initrd')+1]).read_bytes()))
return (1,"Kernel panic - not syncing: IO-APIC + timer doesn't work!") if len(attempts)==1 else original(argv,**options)
sandbox=self.latest(command)
result=sandbox.build(self.run_id,self.source,lambda:False)
self.assertTrue(result['passed'])
self.assertEqual(len(attempts),2)
self.assertEqual(attempts[0],attempts[1])
self.assertIn('noapic',attempts[0][0][-1])
def test_repeated_pre_worker_panic_is_bounded_and_classified(self):
attempts=[]
def command(argv,**options):
attempts.append(argv)
return 1,"Kernel panic - not syncing: IO-APIC + timer doesn't work!"
sandbox=self.latest(command)
with self.assertRaises(SkillError) as error:sandbox.build(self.run_id,self.source,lambda:False)
self.assertEqual(error.exception.code,'SANDBOX_BOOT_FAILED')
self.assertFalse(error.exception.details['generated_code_started'])
self.assertEqual(len(attempts),2)
def test_cancellation_wins_over_boot_recovery(self):
attempts=[];stopped=[False]
def command(argv,**options):
attempts.append(argv);stopped[0]=True
return 1,"Kernel panic - not syncing: IO-APIC + timer doesn't work!"
sandbox=self.latest(command)
with self.assertRaises(SkillError) as error:sandbox.build(self.run_id,self.source,lambda:stopped[0])
self.assertEqual(error.exception.code,'RUN_CANCELLED')
self.assertEqual(len(attempts),1)
def test_post_worker_panic_and_unknown_exit_are_never_replayed(self):
for text in ('ZKCODING_VM_STAGE=compile\nKernel panic - not syncing: IO-APIC + timer doesn\'t work!','unexpected exit'):
with self.subTest(text=text):
attempts=[]
def command(argv,**options):attempts.append(argv);return 1,text
sandbox=self.latest(command)
with self.assertRaises(SkillError) as error:sandbox.build(self.run_id,self.source,lambda:False)
self.assertEqual(error.exception.code,'VM_RESULT_UNKNOWN')
self.assertEqual(len(attempts),1)
def setUp(self):
self.root = Path(self.enterContext(workspace()))
self.image = self.root / 'image'
self.image.mkdir()
(self.image / 'qemu.exe').write_bytes(b'synthetic-executable')
(self.image / 'kernel').write_bytes(b'synthetic-kernel')
(self.image / 'rootfs.cpio.gz').write_bytes(gzip.compress(b''))
control = Path(__file__).resolve().parents[2] / 'modules/execution/container'
paths = {'qemu': self.image / 'qemu.exe', 'kernel': self.image / 'kernel', 'rootfs': self.image / 'rootfs.cpio.gz',
'worker': control / 'vm_worker.py', 'probe': control / 'probe.py'}
manifest = {k: hashlib.sha256(p.read_bytes()).hexdigest() for k, p in paths.items()}
(self.image / 'manifest.json').write_text(json.dumps(manifest))
config = SandboxConfig(image='zkcoding-qemu@sha256:' + digest(manifest), skill_ids=['erp-java-development'],
api_contract='public contract', tests=[{'path': '/health', 'status': 200, 'expected_json': {'status': 'ok'}}],
health={'path': '/health', 'status': 200, 'expected_json': {'status': 'ok'}})
self.calls = []
self.nonce_wrong = False
self.build_fail = False
def command(argv, **options):
self.calls.append(argv)
entries = unpack_overlay(gzip.decompress(Path(argv[argv.index('-initrd') + 1]).read_bytes()))
self.entries = entries
job = json.loads(entries['root/job.json'][1])
report = {'passed': True, 'classes_digest': '1' * 64, 'cases': 1, 'passed_cases': 1}
result = {'nonce': 'wrong' if self.nonce_wrong else job['nonce'],
'build': {'passed': False, 'diagnostics': 'untrusted compile error'} if self.build_fail else report,
'test': report, 'preview': report}
return 0, 'ZKCODING_VM_RESULT=' + json.dumps(result)
self.sandbox = QemuSandbox(self.root / 'runs', config, executable=str(self.image / 'qemu.exe'),
image_root=self.image, command=command)
self.run_id = 'run-' + '1' * 32
self.source = {'files': SOURCE, 'source_digest': digest(SOURCE)}
def test_vm_has_no_network_disk_share_or_model_selected_command(self):
candidate = self.sandbox.build(self.run_id, self.source, lambda: False)
args = self.calls[0]
self.assertEqual(args[args.index('-nic') + 1], 'none')
self.assertEqual(args[args.index('-m') + 1], '1024')
self.assertEqual(args[args.index('-machine') + 1], 'q35,accel=tcg')
for forbidden in ['-drive', '-fsdev', '-virtfs', '-netdev', '-enable-kvm']:
self.assertNotIn(forbidden, args)
self.assertEqual(stat.S_IMODE(self.entries['root/job.json'][0]), 0o600)
self.assertEqual(stat.S_IMODE(self.entries['root'][0]), 0o700)
self.assertFalse(Path(args[args.index('-initrd') + 1]).exists())
self.assertTrue(self.sandbox.probe(self.run_id, candidate, 'test', lambda: False)['passed'])
def test_image_tamper_prevents_launch(self):
(self.image / 'kernel').write_bytes(b'mutated')
with self.assertRaises(SkillError) as err:
self.sandbox.build(self.run_id, self.source, lambda: False)
self.assertEqual(err.exception.code, 'ISOLATION_IMAGE_MISMATCH')
self.assertEqual(self.calls, [])
def test_v6_controller_is_pinned_without_changing_historical_manifest(self):
old=(self.image/'manifest.json').read_bytes()
control=Path(__file__).resolve().parents[2]/'modules/execution/container'
manifest=json.loads(old)
manifest['worker']=hashlib.sha256((control/'vm_worker_v5.py').read_bytes()).hexdigest()
manifest['probe']=hashlib.sha256((control/'probe_v2.py').read_bytes()).hexdigest()
(self.image/'manifest-v6.json').write_text(json.dumps(manifest))
config=self.sandbox.config.model_copy(update={'image':'zkcoding-qemu@sha256:'+digest(manifest)})
latest=QemuSandbox(self.root/'v6',config,executable=self.sandbox.executable,image_root=self.image)
latest.preflight()
self.sandbox.preflight()
self.assertEqual(latest.worker.name,'vm_worker_v5.py')
self.assertEqual(self.sandbox.worker.name,'vm_worker.py')
self.assertEqual((self.image/'manifest.json').read_bytes(),old)
def test_nonce_mismatch_is_unknown_and_not_success(self):
self.nonce_wrong = True
with self.assertRaises(SkillError) as err:
self.sandbox.build(self.run_id, self.source, lambda: False)
self.assertEqual(err.exception.code, 'VM_RESULT_UNKNOWN')
def test_v7_cold_start_limits_are_bounded_and_historical_image_stays_pinned(self):
old=(self.image/'manifest.json').read_bytes()
control=Path(__file__).resolve().parents[2]/'modules/execution/container'
manifest=json.loads(old)
manifest['worker']=hashlib.sha256((control/'vm_worker_v6.py').read_bytes()).hexdigest()
manifest['probe']=hashlib.sha256((control/'probe_v3.py').read_bytes()).hexdigest()
(self.image/'manifest-v7.json').write_text(json.dumps(manifest))
config=self.sandbox.config.model_copy(update={'image':'zkcoding-qemu@sha256:'+digest(manifest)})
latest=QemuSandbox(self.root/'v7',config,executable=self.sandbox.executable,image_root=self.image)
latest.preflight();self.sandbox.preflight()
self.assertEqual(latest.worker.name,'vm_worker_v6.py')
self.assertEqual(latest.command_timeout,300)
self.assertEqual(self.sandbox.command_timeout,150)
self.assertEqual((self.image/'manifest.json').read_bytes(),old)
def test_repairable_compilation_failure_contains_diagnostic(self):
self.build_fail = True
with self.assertRaises(SkillError) as err:
self.sandbox.build(self.run_id, self.source, lambda: False)
self.assertEqual(err.exception.code, 'JAVA_BUILD_FAILED')
self.assertIn('diagnostics', err.exception.details)
def test_receipt_cannot_move_between_runs_or_be_rewritten(self):
candidate = self.sandbox.build(self.run_id, self.source, lambda: False)
with self.assertRaises(SkillError):
self.sandbox.probe('run-' + '2' * 32, candidate, 'test', lambda: False)
path = self.sandbox.root / candidate['receipt_ref'] / 'receipt.json'
body = json.loads(path.read_text())
body['test']['passed_cases'] = 999
path.write_text(json.dumps(body))
with self.assertRaises(SkillError) as err:
self.sandbox.probe(self.run_id, candidate, 'test', lambda: False)
self.assertEqual(err.exception.code, 'EVIDENCE_BINDING_MISMATCH')
def test_source_paths_never_escape_archive_namespace(self):
files = {'../root/job.json': 'malicious'}
with self.assertRaises(SkillError):
self.sandbox.build(self.run_id, {'files': files, 'source_digest': digest(files)}, lambda: False)
self.assertEqual(self.calls, [])