File size: 2,053 Bytes
eeaab36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from pathlib import Path
import json
import matplotlib
matplotlib.use('Agg')
matplotlib.rcParams['svg.fonttype']='none'
import matplotlib.pyplot as plt
import numpy as np
p=Path(__file__).resolve().parent
d=json.loads((p/'BENCHMARKS.json').read_text())
five=d['initial_comparison']['models']['5bit']['cells']
eight=d['qualified_parallel']['models']['8bit']['cells']
fig,axes=plt.subplots(2,2,figsize=(12,7.6));fig.subplots_adjust(top=.86,bottom=.14,hspace=.45,wspace=.28)
x=np.arange(3);width=.34
for row,users in enumerate([1,2]):
 for col,(metric,label) in enumerate([('median_prompt_tps','Prompt processing'),('median_decode_tps','Generation')]):
  ax=axes[row,col]
  for delta,cells,color,name in [(-width/2,five,'#1687d9','5-bit'),(width/2,eight,'#ff8b3d','8-bit')]:
   values=[next(c[metric] for c in cells if c['users']==users and c['input_tokens']==n) for n in [4000,8000,32000]]
   bars=ax.bar(x+delta,values,width,label=name,color=color)
   ax.bar_label(bars,labels=[f'{v:.0f}' if col==0 else f'{v:.1f}' for v in values],padding=3,fontsize=10)
  ax.set_xticks(x,['4K','8K','32K']);ax.set_ylim(0,ax.get_ylim()[1]*1.15)
  ax.set_ylabel('Tokens / second per user');ax.set_title(f'{label} 路 {users} '+('user' if users==1 else 'concurrent users'))
  ax.spines[['top','right']].set_visible(False);ax.grid(axis='y',alpha=.13);ax.set_axisbelow(True)
fig.suptitle('Inkling 2 Mini 路 MLX 5-bit vs 8-bit',fontsize=20,fontweight='bold')
fig.legend(*axes[0,0].get_legend_handles_labels(),loc='upper center',bbox_to_anchor=(.5,.935),ncol=2,frameon=False)
cap=d['refined_capacity']['models']['8bit']
fig.text(.5,.045,f'M5 Max 路 64 GiB 路 fresh KV 路 256 output tokens 路 one attempt per cell 路 no drafting\n8-bit: two full 128K windows passed; peak process footprint {cap["maximum_backend_footprint_gib"]:.1f} GiB, minimum available {cap["minimum_available_gib"]:.1f} GiB, no additional swap.',ha='center',fontsize=10,color='#555555')
fig.savefig(p/'speed-comparison.png',dpi=160,facecolor='white');fig.savefig(p/'speed-comparison.svg',facecolor='white')