"""Reproducible before/after local CPU qualification. Owns every container. Never publishes ports, deploys remotely, or executes selected tools. Expected before runtime files are verified against the exact requested Git commit. """ import argparse import hashlib import json import os from pathlib import Path import subprocess import time def main(): p=argparse.ArgumentParser();p.add_argument('--suites',nargs='+',default=['frozen','heldout','heldout-reserve','coverage-development','coverage-heldout','coverage-adversarial','coverage-reserve']);p.add_argument('--no-tests',action='store_true');p.add_argument('--reuse-baseline',action='store_true');p.add_argument('--before-image',default='needle-gateway-v2:intent-guard');p.add_argument('--after-image',default='needle-gateway-v2:coverage');a=p.parse_args() env=dict(os.environ) for k in ['DOCKER_HOST','DOCKER_CONTEXT','DOCKER_TLS','DOCKER_TLS_VERIFY','DOCKER_CERT_PATH']:env.pop(k,None) D=['docker','--host=unix:///var/run/docker.sock'] def run(args,**kw):return subprocess.run(D+args,env=env,capture_output=True,text=True,check=True,**kw) out=Path('benchmarks/coverage/results');out.mkdir(exist_ok=True) runtime=['gateway.py','models.py','registry.py','laya.py','context_adapter.py','decision_policy.py','request_limits.py'] for label,image,commit in [('before',a.before_image,'9c3db09e3889ea70b2e53e1d4acec2b3a4c5f624'),('after',a.after_image,None)]: start=time.perf_counter() cid=run(['run','-d','--cpus','2','--memory','16g','--network','none','--entrypoint','uvicorn',image,'gateway:app','--app-dir','/srv','--host','127.0.0.1','--port','7860']).stdout.strip() try: checked={} for file in runtime+['intent_guard.py']: expected=subprocess.check_output(['git','show',commit+':'+file]) if commit else Path(file).read_bytes() actual=run(['exec',cid,'sha256sum','/srv/'+file]).stdout.split()[0] assert hashlib.sha256(expected).hexdigest()==actual,(label,file,'source mismatch') checked[file]=actual model_check=run(['exec','--workdir','/root/.cache/cactus-needle/v3/3.0.1',cid,'sha256sum','--check','/srv/needle-assets.sha256']).stdout for folder in ['scripts','benchmarks']:run(['cp',folder,cid+':/srv/'+folder]) ready=False for _ in range(50): r=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True) if not r.returncode:ready=True;break time.sleep(.2) if not ready:raise RuntimeError('Owned test container did not become ready') startup_ms=(time.perf_counter()-start)*1000 for suite in a.suites: if label=='before' and a.reuse_baseline and (out/(label+'-'+suite+'.json')).exists(): cached=json.loads((out/(label+'-'+suite+'.json')).read_text()) assert cached['verified_runtime_source_sha256']==checked fixture=Path('benchmarks/coverage/'+suite.removeprefix('coverage-')+'.json') if suite.startswith('coverage-') else (Path('benchmarks/qualification/scenarios.json') if suite=='frozen' else Path('benchmarks/intent/'+suite+'.json')) assert cached['dataset_sha256']==hashlib.sha256(fixture.read_bytes()).hexdigest() continue if suite in {'heldout-reserve','coverage-development','coverage-heldout','coverage-adversarial','coverage-reserve'}: run(['restart',cid]) for _ in range(50): probe=subprocess.run(D+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True) if not probe.returncode:break time.sleep(.2) else:raise RuntimeError('Reserve-suite restart failed') data=json.loads(run(['exec',cid,'python','-m','scripts.evaluate_coverage','--suite',suite,'--label',label,'--repeat','2','--candidate-check'],timeout=1200).stdout) data['server_memory']=json.loads(run(['exec',cid,'python','-c',"import json;print(json.dumps({l.split(':')[0]:l.split(':')[1].strip() for l in open('/proc/1/status') if l.startswith(('VmHWM:','VmRSS:','Threads:'))}))"]).stdout) data['verified_runtime_source_sha256']=checked data['image_id']=run(['image','inspect','--format','{{.Id}}',image]).stdout.strip() data['model_asset_check']=model_check.strip().splitlines() data['container_start_to_ready_ms']=startup_ms data['resources']={'cpu_quota':2,'memory_gib':16,'network':'none','published_ports':[],'client_in_same_cgroup':True} (out/(label+'-'+suite+'.json')).write_text(json.dumps(data,indent=2)+'\n') print(label,suite,json.dumps(data['summary']),flush=True) finally:run(['rm','-f',cid]) if a.no_tests:return tests=run(['run','--rm','--cpus','2','--memory','16g','--network','none','needle-gateway-v2:coverage-tests']) (out/'offline-cpu-tests.txt').write_text(tests.stdout+tests.stderr) print(tests.stdout,flush=True) if __name__=='__main__':main()