Spaces:
Sleeping
Sleeping
File size: 8,826 Bytes
115b94d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | """Bounded synthetic execution plus actual cross-process resource locking."""
import concurrent.futures
import json
import subprocess
import sys
import threading
import time
import unittest
from pathlib import Path
from src.modules.governance.resource_limits import configured,serial_slot,require_time
from src.modules.llm_gateway.service import ModelReply,ProviderFailure
from src.modules.skill.domain.models import SkillError,digest
from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE
class ResourceLimitTests(unittest.TestCase):
def setUp(self):
self.folder=Path(self.enterContext(workspace()))
self.rt,self.request,self.model,self.sandbox=setup(self.folder)
self.request['resource_limits']=configured()
def active(self,key):
run=self.rt.create(USER,self.request,key)
self.rt.store.mutate(run['id'],lambda r:r.update(status='running',claim='test',claim_started_at=time.time(),lease_until=time.time()+900))
return self.rt.store.get(run['id'])
def test_defaults_and_missing_unbounded_values_are_rejected(self):
p=configured();self.assertEqual((p['max_model_calls'],p['max_repairs'],p['max_features'],p['max_acceptance']),(4,1,3,5))
for change in [{'max_seconds':0},{'max_model_calls':None},{'max_tokens':1000001}]:
with self.assertRaises(ValueError):configured(change)
def test_actual_requests_cannot_exceed_ledger_even_on_format_retry(self):
run=self.active('limited-four-calls')
self.rt.store.mutate(run['id'],lambda r:r['budget'].update(max_model_calls=4))
complete=self.model.complete
def invalid_once(*args):
if self.model.count==0:
self.model.count+=1;return ModelReply({'invalid':'schema'},10,10,self.model.name)
return complete(*args)
self.model.complete=invalid_once
for _ in range(3):self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False)
with self.assertRaises(SkillError) as exc:self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False)
self.assertEqual(exc.exception.code,'MODEL_CALL_BUDGET_EXCEEDED');self.assertEqual(self.model.count,4)
self.assertEqual(len(self.rt.store.calls(run['id'])),4)
self.assertTrue(any(c['metadata'].get('retry_of') for c in self.rt.store.calls(run['id'])))
def test_timeout_precheck_pauses_and_approval_resumes_without_reset(self):
run=self.rt.create(USER,self.request,'limited-time-precheck')
self.rt.store.mutate(run['id'],lambda r:r.update(active_seconds=r['budget']['max_seconds']))
paused=self.rt.advance(USER,run['id']);self.assertEqual(paused['status'],'waiting_limit');self.assertEqual(self.model.count,0)
p=paused['limit_request'];self.assertEqual(p['after']['max_seconds'],1210)
deferred=self.rt.decide_limit(USER,run['id'],p['digest'],False,p['task_revision'])
self.assertEqual(deferred['status'],'waiting_limit');self.assertEqual(deferred['budget'],paused['budget'])
p=deferred['limit_request'];resumed=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision'])
self.assertEqual(resumed['status'],'queued');self.assertEqual(resumed['active_seconds'],900)
again=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision'])
self.assertFalse(again['budget_decision_applied']);self.assertEqual(self.rt.store.calls(run['id']),[])
restored=type(self.rt.store)(self.rt.store.path).get(run['id']);self.assertEqual(restored['budget']['max_seconds'],1210)
def test_one_repair_then_one_retest_stops_with_evidence(self):
self.sandbox.fail='test'
run=self.rt.create(USER,self.request,'one-repair-one-retest')
# Drive the existing design/execution confirmations; no new execution engine.
for _ in range(12):
run=self.rt.store.get(run['id'])
if run['status']=='waiting_limit':break
if run['status']=='queued':self.rt.advance(USER,run['id'])
elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest'])
elif run['status']=='repair_required':self.rt.repair(USER,run['id'])
else:self.fail(run['status'])
run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit')
self.assertEqual(run['repairs'],1);self.assertEqual(self.model.count,3)
self.assertEqual(self.sandbox.calls.count('build'),2);self.assertEqual(self.sandbox.calls.count('test'),2)
self.assertIsNotNone(run['limit_request']);self.assertTrue(run['outputs'])
def test_unknown_timeout_never_replays_or_clears_reservation(self):
run=self.active('unknown-model-timeout')
def unknown(*args):
self.model.count+=1
raise ProviderFailure('MODEL_RESULT_UNKNOWN',details={'failure_category':'transport_timeout'})
self.model.complete=unknown
for _ in range(2):
with self.assertRaises(SkillError):self.rt.llm.generate(run['id'],'code',{'requirement':'fixed'},lambda:False)
calls=self.rt.store.calls(run['id']);self.assertEqual(self.model.count,1)
self.assertEqual(calls[0]['state'],'unknown');self.assertIsNone(calls[0]['charged']);self.assertGreater(calls[0]['reservation'],0)
def test_parallel_models_are_serial_and_queue_is_observed(self):
a=self.active('serial-model-a');b=self.active('serial-model-b');active=0;peak=0;lock=threading.Lock()
def complete(*args):
nonlocal active,peak
with lock:active+=1;peak=max(peak,active)
time.sleep(.1)
with lock:active-=1
return ModelReply({'summary':'bounded'},10,10,self.model.name)
self.model.complete=complete
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
results=list(pool.map(lambda r:self.rt.llm.generate(r['id'],'design',{'requirement':'x'},lambda:False),[a,b]))
self.assertEqual(peak,1);self.assertEqual(len(results),2)
events=self.rt.store.stages(a['id'])+self.rt.store.stages(b['id'])
self.assertTrue(any(e['stage']=='resource_queue' and e['status']=='waiting' for e in events))
def test_heavy_slot_covers_a_separate_process(self):
# Child runs trusted test code only, never generated application code.
code="from pathlib import Path; import sys,time; from src.modules.governance.resource_limits import serial_slot;\nwith serial_slot(Path(sys.argv[1]),'heavy'):\n print('locked',flush=True);time.sleep(.5)"
child=subprocess.Popen([sys.executable,'-c',code,str(self.folder)],stdout=subprocess.PIPE,text=True)
try:
self.assertEqual(child.stdout.readline().strip(),'locked')
with self.assertRaises(SkillError) as exc:
with serial_slot(self.folder,'heavy',timeout=.1):self.fail('overlap')
self.assertEqual(exc.exception.code,'RESOURCE_QUEUE_TIMEOUT')
child.wait(timeout=5)
with serial_slot(self.folder,'heavy',timeout=.1):pass
finally:
if child.poll() is None:child.terminate();child.wait(5)
child.stdout.close()
def test_preview_startup_uses_the_same_heavy_execution_slot(self):
from types import SimpleNamespace
from src.adapters.sandbox.preview import PreviewSession
active=0;peak=0;lock=threading.Lock()
def start():
nonlocal active,peak
with lock:active+=1;peak=max(peak,active)
time.sleep(.1)
with lock:active-=1
sessions=[PreviewSession(SimpleNamespace(resource_root=self.folder,allow_boot_retry=False),{}) for _ in range(2)]
for session in sessions:session._start_unlocked=start
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:list(pool.map(lambda s:s.start(),sessions))
self.assertEqual(peak,1)
def test_environment_failure_does_not_trigger_model_repair(self):
def unavailable(*args):raise SkillError('ISOLATION_UNAVAILABLE','controlled environment failure',503)
self.sandbox.build=unavailable
run=self.rt.create(USER,self.request,'known-environment-failure')
for _ in range(8):
run=self.rt.store.get(run['id'])
if run['status']=='waiting_limit':break
if run['status']=='queued':self.rt.advance(USER,run['id'])
elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest'])
else:self.fail(run['status'])
run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit')
self.assertEqual(self.model.count,2);self.assertEqual(run['repairs'],0)
self.assertIsNone(run['limit_request'])
|