File size: 8,826 Bytes
115b94d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
"""Bounded synthetic execution plus actual cross-process resource locking."""
import concurrent.futures
import json
import subprocess
import sys
import threading
import time
import unittest
from pathlib import Path
from src.modules.governance.resource_limits import configured,serial_slot,require_time
from src.modules.llm_gateway.service import ModelReply,ProviderFailure
from src.modules.skill.domain.models import SkillError,digest
from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE


class ResourceLimitTests(unittest.TestCase):
    def setUp(self):
        self.folder=Path(self.enterContext(workspace()))
        self.rt,self.request,self.model,self.sandbox=setup(self.folder)
        self.request['resource_limits']=configured()

    def active(self,key):
        run=self.rt.create(USER,self.request,key)
        self.rt.store.mutate(run['id'],lambda r:r.update(status='running',claim='test',claim_started_at=time.time(),lease_until=time.time()+900))
        return self.rt.store.get(run['id'])

    def test_defaults_and_missing_unbounded_values_are_rejected(self):
        p=configured();self.assertEqual((p['max_model_calls'],p['max_repairs'],p['max_features'],p['max_acceptance']),(4,1,3,5))
        for change in [{'max_seconds':0},{'max_model_calls':None},{'max_tokens':1000001}]:
            with self.assertRaises(ValueError):configured(change)

    def test_actual_requests_cannot_exceed_ledger_even_on_format_retry(self):
        run=self.active('limited-four-calls')
        self.rt.store.mutate(run['id'],lambda r:r['budget'].update(max_model_calls=4))
        complete=self.model.complete
        def invalid_once(*args):
            if self.model.count==0:
                self.model.count+=1;return ModelReply({'invalid':'schema'},10,10,self.model.name)
            return complete(*args)
        self.model.complete=invalid_once
        for _ in range(3):self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False)
        with self.assertRaises(SkillError) as exc:self.rt.llm.generate(run['id'],'design',{'requirement':'fixed'},lambda:False)
        self.assertEqual(exc.exception.code,'MODEL_CALL_BUDGET_EXCEEDED');self.assertEqual(self.model.count,4)
        self.assertEqual(len(self.rt.store.calls(run['id'])),4)
        self.assertTrue(any(c['metadata'].get('retry_of') for c in self.rt.store.calls(run['id'])))

    def test_timeout_precheck_pauses_and_approval_resumes_without_reset(self):
        run=self.rt.create(USER,self.request,'limited-time-precheck')
        self.rt.store.mutate(run['id'],lambda r:r.update(active_seconds=r['budget']['max_seconds']))
        paused=self.rt.advance(USER,run['id']);self.assertEqual(paused['status'],'waiting_limit');self.assertEqual(self.model.count,0)
        p=paused['limit_request'];self.assertEqual(p['after']['max_seconds'],1210)
        deferred=self.rt.decide_limit(USER,run['id'],p['digest'],False,p['task_revision'])
        self.assertEqual(deferred['status'],'waiting_limit');self.assertEqual(deferred['budget'],paused['budget'])
        p=deferred['limit_request'];resumed=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision'])
        self.assertEqual(resumed['status'],'queued');self.assertEqual(resumed['active_seconds'],900)
        again=self.rt.decide_limit(USER,run['id'],p['digest'],True,p['task_revision'])
        self.assertFalse(again['budget_decision_applied']);self.assertEqual(self.rt.store.calls(run['id']),[])
        restored=type(self.rt.store)(self.rt.store.path).get(run['id']);self.assertEqual(restored['budget']['max_seconds'],1210)

    def test_one_repair_then_one_retest_stops_with_evidence(self):
        self.sandbox.fail='test'
        run=self.rt.create(USER,self.request,'one-repair-one-retest')
        # Drive the existing design/execution confirmations; no new execution engine.
        for _ in range(12):
            run=self.rt.store.get(run['id'])
            if run['status']=='waiting_limit':break
            if run['status']=='queued':self.rt.advance(USER,run['id'])
            elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest'])
            elif run['status']=='repair_required':self.rt.repair(USER,run['id'])
            else:self.fail(run['status'])
        run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit')
        self.assertEqual(run['repairs'],1);self.assertEqual(self.model.count,3)
        self.assertEqual(self.sandbox.calls.count('build'),2);self.assertEqual(self.sandbox.calls.count('test'),2)
        self.assertIsNotNone(run['limit_request']);self.assertTrue(run['outputs'])

    def test_unknown_timeout_never_replays_or_clears_reservation(self):
        run=self.active('unknown-model-timeout')
        def unknown(*args):
            self.model.count+=1
            raise ProviderFailure('MODEL_RESULT_UNKNOWN',details={'failure_category':'transport_timeout'})
        self.model.complete=unknown
        for _ in range(2):
            with self.assertRaises(SkillError):self.rt.llm.generate(run['id'],'code',{'requirement':'fixed'},lambda:False)
        calls=self.rt.store.calls(run['id']);self.assertEqual(self.model.count,1)
        self.assertEqual(calls[0]['state'],'unknown');self.assertIsNone(calls[0]['charged']);self.assertGreater(calls[0]['reservation'],0)

    def test_parallel_models_are_serial_and_queue_is_observed(self):
        a=self.active('serial-model-a');b=self.active('serial-model-b');active=0;peak=0;lock=threading.Lock()
        def complete(*args):
            nonlocal active,peak
            with lock:active+=1;peak=max(peak,active)
            time.sleep(.1)
            with lock:active-=1
            return ModelReply({'summary':'bounded'},10,10,self.model.name)
        self.model.complete=complete
        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
            results=list(pool.map(lambda r:self.rt.llm.generate(r['id'],'design',{'requirement':'x'},lambda:False),[a,b]))
        self.assertEqual(peak,1);self.assertEqual(len(results),2)
        events=self.rt.store.stages(a['id'])+self.rt.store.stages(b['id'])
        self.assertTrue(any(e['stage']=='resource_queue' and e['status']=='waiting' for e in events))

    def test_heavy_slot_covers_a_separate_process(self):
        # Child runs trusted test code only, never generated application code.
        code="from pathlib import Path; import sys,time; from src.modules.governance.resource_limits import serial_slot;\nwith serial_slot(Path(sys.argv[1]),'heavy'):\n print('locked',flush=True);time.sleep(.5)"
        child=subprocess.Popen([sys.executable,'-c',code,str(self.folder)],stdout=subprocess.PIPE,text=True)
        try:
            self.assertEqual(child.stdout.readline().strip(),'locked')
            with self.assertRaises(SkillError) as exc:
                with serial_slot(self.folder,'heavy',timeout=.1):self.fail('overlap')
            self.assertEqual(exc.exception.code,'RESOURCE_QUEUE_TIMEOUT')
            child.wait(timeout=5)
            with serial_slot(self.folder,'heavy',timeout=.1):pass
        finally:
            if child.poll() is None:child.terminate();child.wait(5)
            child.stdout.close()

    def test_preview_startup_uses_the_same_heavy_execution_slot(self):
        from types import SimpleNamespace
        from src.adapters.sandbox.preview import PreviewSession
        active=0;peak=0;lock=threading.Lock()
        def start():
            nonlocal active,peak
            with lock:active+=1;peak=max(peak,active)
            time.sleep(.1)
            with lock:active-=1
        sessions=[PreviewSession(SimpleNamespace(resource_root=self.folder,allow_boot_retry=False),{}) for _ in range(2)]
        for session in sessions:session._start_unlocked=start
        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:list(pool.map(lambda s:s.start(),sessions))
        self.assertEqual(peak,1)

    def test_environment_failure_does_not_trigger_model_repair(self):
        def unavailable(*args):raise SkillError('ISOLATION_UNAVAILABLE','controlled environment failure',503)
        self.sandbox.build=unavailable
        run=self.rt.create(USER,self.request,'known-environment-failure')
        for _ in range(8):
            run=self.rt.store.get(run['id'])
            if run['status']=='waiting_limit':break
            if run['status']=='queued':self.rt.advance(USER,run['id'])
            elif run['status']=='waiting_approval':self.rt.approve(USER,run['id'],run['pending_approval']['digest'])
            else:self.fail(run['status'])
        run=self.rt.store.get(run['id']);self.assertEqual(run['status'],'waiting_limit')
        self.assertEqual(self.model.count,2);self.assertEqual(run['repairs'],0)
        self.assertIsNone(run['limit_request'])