Spaces:
Running
Running
File size: 5,375 Bytes
25a2c1c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Reproducible before/after local CPU qualification. Owns every container.
Never publishes ports, deploys remotely, or executes selected tools. Expected
before runtime files are verified against the exact requested Git commit.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
def main():
p=argparse.ArgumentParser();p.add_argument('--suites',nargs='+',default=['frozen','heldout','heldout-reserve','coverage-development','coverage-heldout','coverage-adversarial','coverage-reserve']);p.add_argument('--no-tests',action='store_true');p.add_argument('--reuse-baseline',action='store_true');p.add_argument('--before-image',default='needle-gateway-v2:intent-guard');p.add_argument('--after-image',default='needle-gateway-v2:coverage');a=p.parse_args()
env=dict(os.environ)
for k in ['DOCKER_HOST','DOCKER_CONTEXT','DOCKER_TLS','DOCKER_TLS_VERIFY','DOCKER_CERT_PATH']:env.pop(k,None)
D=['docker','--host=unix:///var/run/docker.sock']
def run(args,**kw):return subprocess.run(D+args,env=env,capture_output=True,text=True,check=True,**kw)
out=Path('benchmarks/coverage/results');out.mkdir(exist_ok=True)
runtime=['gateway.py','models.py','registry.py','laya.py','context_adapter.py','decision_policy.py','request_limits.py']
for label,image,commit in [('before',a.before_image,'9c3db09e3889ea70b2e53e1d4acec2b3a4c5f624'),('after',a.after_image,None)]:
start=time.perf_counter()
cid=run(['run','-d','--cpus','2','--memory','16g','--network','none','--entrypoint','uvicorn',image,'gateway:app','--app-dir','/srv','--host','127.0.0.1','--port','7860']).stdout.strip()
try:
checked={}
for file in runtime+['intent_guard.py']:
expected=subprocess.check_output(['git','show',commit+':'+file]) if commit else Path(file).read_bytes()
actual=run(['exec',cid,'sha256sum','/srv/'+file]).stdout.split()[0]
assert hashlib.sha256(expected).hexdigest()==actual,(label,file,'source mismatch')
checked[file]=actual
model_check=run(['exec','--workdir','/root/.cache/cactus-needle/v3/3.0.1',cid,'sha256sum','--check','/srv/needle-assets.sha256']).stdout
for folder in ['scripts','benchmarks']:run(['cp',folder,cid+':/srv/'+folder])
ready=False
for _ in range(50):
r=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True)
if not r.returncode:ready=True;break
time.sleep(.2)
if not ready:raise RuntimeError('Owned test container did not become ready')
startup_ms=(time.perf_counter()-start)*1000
for suite in a.suites:
if label=='before' and a.reuse_baseline and (out/(label+'-'+suite+'.json')).exists():
cached=json.loads((out/(label+'-'+suite+'.json')).read_text())
assert cached['verified_runtime_source_sha256']==checked
fixture=Path('benchmarks/coverage/'+suite.removeprefix('coverage-')+'.json') if suite.startswith('coverage-') else (Path('benchmarks/qualification/scenarios.json') if suite=='frozen' else Path('benchmarks/intent/'+suite+'.json'))
assert cached['dataset_sha256']==hashlib.sha256(fixture.read_bytes()).hexdigest()
continue
if suite in {'heldout-reserve','coverage-development','coverage-heldout','coverage-adversarial','coverage-reserve'}:
run(['restart',cid])
for _ in range(50):
probe=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True)
if not probe.returncode:break
time.sleep(.2)
else:raise RuntimeError('Reserve-suite restart failed')
data=json.loads(run(['exec',cid,'python','-m','scripts.evaluate_coverage','--suite',suite,'--label',label,'--repeat','2','--candidate-check'],timeout=1200).stdout)
data['server_memory']=json.loads(run(['exec',cid,'python','-c',"import json;print(json.dumps({l.split(':')[0]:l.split(':')[1].strip() for l in open('/proc/1/status') if l.startswith(('VmHWM:','VmRSS:','Threads:'))}))"]).stdout)
data['verified_runtime_source_sha256']=checked
data['image_id']=run(['image','inspect','--format','{{.Id}}',image]).stdout.strip()
data['model_asset_check']=model_check.strip().splitlines()
data['container_start_to_ready_ms']=startup_ms
data['resources']={'cpu_quota':2,'memory_gib':16,'network':'none','published_ports':[],'client_in_same_cgroup':True}
(out/(label+'-'+suite+'.json')).write_text(json.dumps(data,indent=2)+'\n')
print(label,suite,json.dumps(data['summary']),flush=True)
finally:run(['rm','-f',cid])
if a.no_tests:return
tests=run(['run','--rm','--cpus','2','--memory','16g','--network','none','needle-gateway-v2:coverage-tests'])
(out/'offline-cpu-tests.txt').write_text(tests.stdout+tests.stderr)
print(tests.stdout,flush=True)
if __name__=='__main__':main()
|