File size: 3,743 Bytes
6c2b324 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | from pathlib import Path
import json
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
out=Path(__file__).resolve().parent.parent
base=json.loads((out/"benchmarks/original_base_polynomials.json").read_text())["tasks"]
orig=next(x for x in json.loads((out/"benchmarks/original_slerp_sweep.json").read_text())["rows"] if x["tag"]=="t080")
rep=json.loads((out/"benchmarks/alice_reproduction.json").read_text())
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':11,'axes.spines.top':False,'axes.spines.right':False,'axes.spines.left':False,'axes.titleweight':'bold','axes.labelcolor':'#475569','xtick.color':'#334155','ytick.color':'#475569','figure.facecolor':'white','axes.facecolor':'white','savefig.facecolor':'white'})
colors=['#64748b','#2563eb','#0f9d8a'];fig,axs=plt.subplots(2,2,figsize=(13,8.6));fig.subplots_adjust(top=.84,bottom=.14,left=.07,right=.98,hspace=.55,wspace=.25)
fig.suptitle('BestTerm-440M vs. NextTerm-440M',x=.07,y=.97,ha='left',fontsize=23,fontweight='bold',color='#0f172a')
fig.text(.07,.918,'Stronger short-sequence completion; broadly preserved OEIS and polynomial accuracy.',fontsize=12,color='#475569')
for ax in axs.flat:ax.set_axisbelow(True);ax.grid(axis='y',color='#e2e8f0');ax.tick_params(axis='both',length=0)
ax=axs[0,0];x=np.arange(2)
for offset,vals,color,label in [(-.18,[30/57*100,32/57*100],colors[0],'NextTerm-440M'),(.18,[38/57*100,40/57*100],colors[1],'BestTerm-440M')]:
bars=ax.bar(x+offset,vals,.32,color=color,label=label);ax.bar_label(bars,labels=[f'{v:.2f}%' for v in vals],padding=4,fontsize=10)
ax.set_xticks(x,['Greedy','Beam-4']);ax.set_ylim(0,85);ax.set_title('Ryskina & Knight 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)');ax.legend(frameon=False,fontsize=9,loc='upper left',ncol=2)
labels=['Base','BestTerm\noriginal','BestTerm\nSpark']
ax=axs[0,1];vals=[6555/19034*100,6532/19034*100,6536/19034*100];bars=ax.bar(range(3),vals,.6,color=colors);ax.bar_label(bars,labels=[f'{v:.3f}%' for v in vals],padding=4);ax.set_xticks(range(3),labels);ax.set_ylim(0,42);ax.set_title('OEIS-Eval-Neo 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)')
ax=axs[1,0];vals=[17.623926615,17.582547764,17.813680040];bars=ax.bar(range(3),vals,.6,color=colors);ax.bar_label(bars,labels=[f'{v:.4f}' for v in vals],padding=4);ax.set_xticks(range(3),labels);ax.set_ylim(0,22);ax.set_title('M1 Competition 111 路 lower is better',loc='left',pad=12);ax.set_ylabel('Macro MAPE');ax.axhline(17.798673617,color='#b45309',linestyle='--',linewidth=1.2);ax.text(2.4,20.1,'Naive2: 17.7987',ha='right',color='#92400e',fontsize=10)
ax=axs[1,1];tasks=['arithmetic','quadratic','cubic','quartic'];x=np.arange(4)
for offset,vals,color,label in [(-.24,[100*base[k]['accuracy'] for k in tasks],colors[0],'Base'),(0,[100*orig['poly_'+k] for k in tasks],colors[1],'BestTerm original'),(.24,[100*rep['poly_'+k] for k in tasks],colors[2],'BestTerm Spark')]:ax.bar(x+offset,vals,.22,color=color,label=label)
ax.set_xticks(x,['Arithmetic','Quadratic','Cubic','Quartic']);ax.set_ylim(0,110);ax.set_title('Polynomial continuation 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)');ax.legend(frameon=False,fontsize=8,ncol=3,loc='upper center')
fig.text(.07,.06,'Ryskina: matched Transformers comparison. Other panels: original experiment controls and saved Spark reproduction.\nHistorical July 2026 results; recovered September 2026. Selection used these benchmarks. See model card for protocols.',fontsize=10,color='#64748b',linespacing=1.6)
fig.savefig(out/'assets/benchmark_comparison.png',dpi=180);plt.close(fig)
print("Saved assets/benchmark_comparison.png", flush=True)
|