scripts/plot_benchmarks.py

#!/usr/bin/env python3
"""Plot actual observations; no fitted/synthetic latency values."""
from pathlib import Path
import json
import statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

root=Path(__file__).resolve().parents[1]/'artifacts/benchmarks'
rows=[json.loads(l) for l in (root/'20260920T205217-steps/results.jsonl').read_text().splitlines()]
fig,ax=plt.subplots(1,2,figsize=(11,4.7),gridspec_kw={'width_ratios':[1.3,1]})
colors={True:'#777d60',False:'#9c6c28'}
for steps in [20,30,40]:
 selected=[r for r in rows if r['recipe']['parameters']['steps']==steps]
 vals=[]
 for i,r in enumerate(selected):
  m=r['recipe']['metrics'];y=m['inference_seconds'];vals.append(y)
  warm=m.get('gpu_worker_call_index',1)>1
  ax[0].scatter(steps+(i-3.5)*.33,y,color=colors[warm],s=35)
 ax[0].plot([steps-2,steps+2],[statistics.median(vals)]*2,color='#252521',lw=2)
ax[0].set(xticks=[20,30,40],xlabel='Denoising steps',ylabel='Synchronized pipeline seconds',
          title='48 GB · 1K · eight prompt/seed cases per step',ylim=(0,26))
for x,(label,suite) in enumerate([('KV on','20260920T212255-repeatability'),('KV off','20260920T214119-repeatability')]):
 rs=[json.loads(l) for l in (root/suite/'results.jsonl').read_text().splitlines()]
 for i,r in enumerate(rs):
  m=r['recipe']['metrics'];warm=m['gpu_worker_call_index']>1
  ax[1].scatter(x+(i-1)*.07,m['inference_seconds'],color=colors[warm],s=45)
ax[1].set(xticks=[0,1],xticklabels=['KV on','KV off'],xlim=(-.5,1.5),ylim=(0,14),
          ylabel='Synchronized pipeline seconds',title='96 GB · 1K / 40 · same text prompt and seed')
for a in ax:
 a.spines[['top','right']].set_visible(False);a.grid(axis='y',alpha=.15);a.set_axisbelow(True)
handles=[plt.Line2D([0],[0],marker='o',linestyle='',color=colors[v],label=l) for v,l in [(True,'Reused GPU worker'),(False,'New GPU worker')]]
fig.legend(handles=handles,loc='lower center',ncol=2,frameon=False,bbox_to_anchor=(.5,.06))
fig.text(.5,.015,'Actual ZeroGPU measurements, 20 September 2026. Small samples; no population confidence interval or latency SLA.',ha='center',fontsize=9,color='#666')
fig.tight_layout(rect=[0,.15,1,1]);fig.savefig(root/'latency.png',dpi=180);fig.savefig(root/'latency.svg')
print(root/'latency.png')