Spaces:
Running
Running
File size: 3,802 Bytes
25a2c1c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Run owned, offline local Docker qualification containers. Never deploys.
Build before/final images from the documented commits first. Keeps Docker config
and registry credentials intact. No ports are published. Test clients run inside
the same 2-CPU cgroup, identically across versions.
"""
import argparse
import io
import json
import os
from pathlib import Path
import subprocess
import tarfile
import tempfile
import time
def main():
p=argparse.ArgumentParser();p.add_argument('--before-image',required=True);p.add_argument('--final-image',required=True);p.add_argument('--only-final',action='store_true');p.add_argument('--output',default='benchmarks/results/final');args=p.parse_args()
env=dict(os.environ)
for key in ['DOCKER_HOST','DOCKER_CONTEXT','DOCKER_TLS','DOCKER_TLS_VERIFY','DOCKER_CERT_PATH']:env.pop(key,None)
docker=['docker','--host=unix:///var/run/docker.sock']
def run(argv,**kwargs):return subprocess.run(docker+argv,env=env,check=True,capture_output=True,text=True,**kwargs)
out=Path(args.output);out.mkdir(parents=True,exist_ok=True)
with tempfile.TemporaryDirectory(prefix='needle-original-') as tmp:
archive=subprocess.check_output(['git','archive','f7fbfaabc040e1e765f881c01360e2e091f90a51'])
with tarfile.open(fileobj=io.BytesIO(archive)) as tf:tf.extractall(tmp,filter='data')
jobs=[] if args.only_final else [('original',args.before_image,tmp,1,1,True),('v2-before',args.before_image,None,1,2,True)]
jobs += [('v2-final',args.final_image,None,1,2,True),('v2-final-c2',args.final_image,None,2,1,False),('v2-final-c4',args.final_image,None,4,1,False)]
for label,image,source,concurrency,repeat,candidates in jobs:
start=time.perf_counter()
cid=run(['run','-d','--cpus','2','--memory','16g','--network','none','--entrypoint','uvicorn',image,'gateway:app','--app-dir','/srv','--host','127.0.0.1','--port','7860']).stdout.strip()
try:
if source:
for f in ['gateway.py','models.py','registry.py','laya.py']:run(['cp',str(Path(source)/f),cid+':/srv/'+f])
run(['restart',cid]);start=time.perf_counter()
for f in ['scripts','benchmarks']:run(['cp',f,cid+':/srv/'+f])
ready=False
for _ in range(50):
r=subprocess.run(docker+['exec',cid,'python','-c',"import urllib.request;urllib.request.urlopen('http://127.0.0.1:7860/ready')"],env=env,capture_output=True)
if r.returncode==0:ready=True;break
time.sleep(.2)
if not ready:raise RuntimeError('Owned qualification container did not become ready')
ready_ms=(time.perf_counter()-start)*1000
command=['exec',cid,'python','-m','scripts.final_qualification','--label',label,'--concurrency',str(concurrency),'--repeat',str(repeat)]
if candidates:command+=['--candidate-check']
data=json.loads(run(command,timeout=1200).stdout)
data['server_memory']=json.loads(run(['exec',cid,'python','-c',"import json;print(json.dumps({l.split(':')[0]:l.split(':')[1].strip() for l in open('/proc/1/status') if l.startswith(('VmHWM:','VmRSS:','Threads:'))}))"]).stdout)
data['image_id']=run(['image','inspect','--format','{{.Id}}',image]).stdout.strip()
data['container_start_to_ready_ms']=ready_ms
data['resources']={'cpu_quota':2,'memory_gib':16,'network':'none','published_ports':[],'client_in_same_cgroup':True}
(out/(label+'.json')).write_text(json.dumps(data,indent=2)+'\n')
print(label,json.dumps(data['summary']),flush=True)
finally:run(['rm','-f',cid])
if __name__=='__main__':main()
|