Spaces:
Running
Running
| """Reproducible before/after local CPU qualification. Owns every container. | |
| Never publishes ports, deploys remotely, or executes selected tools. Expected | |
| before runtime files are verified against the exact requested Git commit. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import time | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--reuse-baseline',action='store_true');p.add_argument('--before-image',default='needle-gateway-v2:final-qualification');p.add_argument('--after-image',default='needle-gateway-v2:intent-guard');a=p.parse_args() | |
| env=dict(os.environ) | |
| for k in ['DOCKER_HOST','DOCKER_CONTEXT','DOCKER_TLS','DOCKER_TLS_VERIFY','DOCKER_CERT_PATH']:env.pop(k,None) | |
| D=['docker','--host=unix:///var/run/docker.sock'] | |
| def run(args,**kw):return subprocess.run(D+args,env=env,capture_output=True,text=True,check=True,**kw) | |
| out=Path('benchmarks/intent/results');out.mkdir(exist_ok=True) | |
| runtime=['gateway.py','models.py','registry.py','laya.py','context_adapter.py','decision_policy.py','request_limits.py'] | |
| for label,image,commit in [('before',a.before_image,'a165f4cc684b23a0fb853ded7aac3a2498c53c61'),('after',a.after_image,None)]: | |
| start=time.perf_counter() | |
| cid=run(['run','-d','--cpus','2','--memory','16g','--network','none','--entrypoint','uvicorn',image,'gateway:app','--app-dir','/srv','--host','127.0.0.1','--port','7860']).stdout.strip() | |
| try: | |
| checked={} | |
| for file in runtime+(['intent_guard.py'] if label=='after' else []): | |
| expected=subprocess.check_output(['git','show',commit+':'+file]) if commit else Path(file).read_bytes() | |
| actual=run(['exec',cid,'sha256sum','/srv/'+file]).stdout.split()[0] | |
| assert hashlib.sha256(expected).hexdigest()==actual,(label,file,'source mismatch') | |
| checked[file]=actual | |
| model_check=run(['exec','--workdir','/root/.cache/cactus-needle/v3/3.0.1',cid,'sha256sum','--check','/srv/needle-assets.sha256']).stdout | |
| for folder in ['scripts','benchmarks']:run(['cp',folder,cid+':/srv/'+folder]) | |
| ready=False | |
| for _ in range(50): | |
| r=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True) | |
| if not r.returncode:ready=True;break | |
| time.sleep(.2) | |
| if not ready:raise RuntimeError('Owned test container did not become ready') | |
| startup_ms=(time.perf_counter()-start)*1000 | |
| for suite in ['frozen','heldout','heldout-reserve']: | |
| if label=='before' and a.reuse_baseline and (out/(label+'-'+suite+'.json')).exists(): | |
| cached=json.loads((out/(label+'-'+suite+'.json')).read_text()) | |
| assert cached['verified_runtime_source_sha256']==checked | |
| fixture=Path('benchmarks/qualification/scenarios.json') if suite=='frozen' else Path('benchmarks/intent/'+suite+'.json') | |
| assert cached['dataset_sha256']==hashlib.sha256(fixture.read_bytes()).hexdigest() | |
| continue | |
| if suite=='heldout-reserve': | |
| run(['restart',cid]) | |
| for _ in range(50): | |
| probe=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True) | |
| if not probe.returncode:break | |
| time.sleep(.2) | |
| else:raise RuntimeError('Reserve-suite restart failed') | |
| data=json.loads(run(['exec',cid,'python','-m','scripts.evaluate_intent','--suite',suite,'--label',label,'--repeat','2','--candidate-check'],timeout=1200).stdout) | |
| data['server_memory']=json.loads(run(['exec',cid,'python','-c',"import json;print(json.dumps({l.split(':')[0]:l.split(':')[1].strip() for l in open('/proc/1/status') if l.startswith(('VmHWM:','VmRSS:','Threads:'))}))"]).stdout) | |
| data['verified_runtime_source_sha256']=checked | |
| data['image_id']=run(['image','inspect','--format','{{.Id}}',image]).stdout.strip() | |
| data['model_asset_check']=model_check.strip().splitlines() | |
| data['container_start_to_ready_ms']=startup_ms | |
| data['resources']={'cpu_quota':2,'memory_gib':16,'network':'none','published_ports':[],'client_in_same_cgroup':True} | |
| (out/(label+'-'+suite+'.json')).write_text(json.dumps(data,indent=2)+'\n') | |
| print(label,suite,json.dumps(data['summary']),flush=True) | |
| finally:run(['rm','-f',cid]) | |
| tests=run(['run','--rm','--cpus','2','--memory','16g','--network','none','needle-gateway-v2:intent-tests']) | |
| (out/'offline-cpu-tests.txt').write_text(tests.stdout+tests.stderr) | |
| print(tests.stdout,flush=True) | |
| if __name__=='__main__':main() | |