"""Bounded synthetic execution plus actual cross-process resource locking.""" import concurrent.futures import json import subprocess import sys import threading import time import unittest from pathlib import Path from src.modules.governance.resource_limits import configured,serial_slot,require_time from src.modules.llm_gateway.service import ModelReply,ProviderFailure from src.modules.skill.domain.models import SkillError,digest from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE class ResourceLimitTests(unittest.TestCase): def setUp(self): self.folder=Path(self.enterContext(workspace())) self.rt,self.request,self.model,self.sandbox=setup(self.folder) self.request['resource_limits']=configured() def active(self,key): run=self.rt.create(USER,self.request,key) self.rt.store.mutate(run['id'],lambda r:r.update(status='running',claim='test',claim_started_at=time.time(),lease_until=time.time()+900)) return self.rt.store.get(run['id']) def test_defaults_and_missing_unbounded_values_are_rejected(self): p=configured();self.assertEqual((p['max_model_calls'],p['max_repairs'],p['max_features'],p['max_acceptance']),(4,1,3,5)) for change in [{'max_seconds':0},{'max_model_calls':None},{'max_tokens':1000001}]: with self.assertRaises(ValueError):configured(change) def test_actual_requests_cannot_exceed_ledger_even_on_format_retry(self): run=self.active('limited-four-calls') self.rt.store.mutate(run['id'],lambda r:r['budget'].update(max_model_calls=4)) complete=self.model.complete def invalid_once(*args): if self.model.count==0: self.model.count+=1;return ModelReply({'invalid':'schema'},10,10,self.model.name) return complete(*args) self.model.complete=invalid_once for _ in range(3):self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False) with self.assertRaises(SkillError) as exc:self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False) self.assertEqual(exc.exception.code,'MODEL_CALL_BUDGET_EXCEEDED');self.assertEqual(self.model.count,4) self.assertEqual(len(self.rt.store.calls(run['id'])),4) self.assertTrue(any(c['metadata'].get('retry_of') for c in self.rt.store.calls(run['id']))) def test_timeout_precheck_pauses_and_approval_resumes_without_reset(self): run=self.rt.create(USER,self.request,'limited-time-precheck') self.rt.store.mutate(run['id'],lambda r:r.update(active_seconds=r['budget']['max_seconds'])) paused=self.rt.advance(USER,run['id']);self.assertEqual(paused['status'],'waiting_limit');self.assertEqual(self.model.count,0) p=paused['limit_request'];self.assertEqual(p['after']['max_seconds'],1210) deferred=self.rt.decide_limit(USER,run['id'],p['digest'],False,p['task_revision']) self.assertEqual(deferred['status'],'waiting_limit');self.assertEqual(deferred['budget'],paused['budget']) p=deferred['limit_request'];resumed=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision']) self.assertEqual(resumed['status'],'queued');self.assertEqual(resumed['active_seconds'],900) again=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision']) self.assertFalse(again['budget_decision_applied']);self.assertEqual(self.rt.store.calls(run['id']),[]) restored=type(self.rt.store)(self.rt.store.path).get(run['id']);self.assertEqual(restored['budget']['max_seconds'],1210) def test_one_repair_then_one_retest_stops_with_evidence(self): self.sandbox.fail='test' run=self.rt.create(USER,self.request,'one-repair-one-retest') # Drive the existing design/execution confirmations; no new execution engine. for _ in range(12): run=self.rt.store.get(run['id']) if run['status']=='waiting_limit':break if run['status']=='queued':self.rt.advance(USER,run['id']) elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest']) elif run['status']=='repair_required':self.rt.repair(USER,run['id']) else:self.fail(run['status']) run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit') self.assertEqual(run['repairs'],1);self.assertEqual(self.model.count,3) self.assertEqual(self.sandbox.calls.count('build'),2);self.assertEqual(self.sandbox.calls.count('test'),2) self.assertIsNotNone(run['limit_request']);self.assertTrue(run['outputs']) def test_unknown_timeout_never_replays_or_clears_reservation(self): run=self.active('unknown-model-timeout') def unknown(*args): self.model.count+=1 raise ProviderFailure('MODEL_RESULT_UNKNOWN',details={'failure_category':'transport_timeout'}) self.model.complete=unknown for _ in range(2): with self.assertRaises(SkillError):self.rt.llm.generate(run['id'],'code',{'requirement':'fixed'},lambda:False) calls=self.rt.store.calls(run['id']);self.assertEqual(self.model.count,1) self.assertEqual(calls[0]['state'],'unknown');self.assertIsNone(calls[0]['charged']);self.assertGreater(calls[0]['reservation'],0) def test_parallel_models_are_serial_and_queue_is_observed(self): a=self.active('serial-model-a');b=self.active('serial-model-b');active=0;peak=0;lock=threading.Lock() def complete(*args): nonlocal active,peak with lock:active+=1;peak=max(peak,active) time.sleep(.1) with lock:active-=1 return ModelReply({'summary':'bounded'},10,10,self.model.name) self.model.complete=complete with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda r:self.rt.llm.generate(r['id'],'design',{'requirement':'x'},lambda:False),[a,b])) self.assertEqual(peak,1);self.assertEqual(len(results),2) events=self.rt.store.stages(a['id'])+self.rt.store.stages(b['id']) self.assertTrue(any(e['stage']=='resource_queue' and e['status']=='waiting' for e in events)) def test_heavy_slot_covers_a_separate_process(self): # Child runs trusted test code only, never generated application code. code="from pathlib import Path; import sys,time; from src.modules.governance.resource_limits import serial_slot;\nwith serial_slot(Path(sys.argv[1]),'heavy'):\n print('locked',flush=True);time.sleep(.5)" child=subprocess.Popen([sys.executable,'-c',code,str(self.folder)],stdout=subprocess.PIPE,text=True) try: self.assertEqual(child.stdout.readline().strip(),'locked') with self.assertRaises(SkillError) as exc: with serial_slot(self.folder,'heavy',timeout=.1):self.fail('overlap') self.assertEqual(exc.exception.code,'RESOURCE_QUEUE_TIMEOUT') child.wait(timeout=5) with serial_slot(self.folder,'heavy',timeout=.1):pass finally: if child.poll() is None:child.terminate();child.wait(5) child.stdout.close() def test_preview_startup_uses_the_same_heavy_execution_slot(self): from types import SimpleNamespace from src.adapters.sandbox.preview import PreviewSession active=0;peak=0;lock=threading.Lock() def start(): nonlocal active,peak with lock:active+=1;peak=max(peak,active) time.sleep(.1) with lock:active-=1 sessions=[PreviewSession(SimpleNamespace(resource_root=self.folder,allow_boot_retry=False),{}) for _ in range(2)] for session in sessions:session._start_unlocked=start with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:list(pool.map(lambda s:s.start(),sessions)) self.assertEqual(peak,1) def test_environment_failure_does_not_trigger_model_repair(self): def unavailable(*args):raise SkillError('ISOLATION_UNAVAILABLE','controlled environment failure',503) self.sandbox.build=unavailable run=self.rt.create(USER,self.request,'known-environment-failure') for _ in range(8): run=self.rt.store.get(run['id']) if run['status']=='waiting_limit':break if run['status']=='queued':self.rt.advance(USER,run['id']) elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest']) else:self.fail(run['status']) run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit') self.assertEqual(self.model.count,2);self.assertEqual(run['repairs'],0) self.assertIsNone(run['limit_request'])