| from pathlib import Path |
| import json |
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
| import numpy as np |
| out=Path(__file__).resolve().parent.parent |
| base=json.loads((out/"benchmarks/original_base_polynomials.json").read_text())["tasks"] |
| orig=next(x for x in json.loads((out/"benchmarks/original_slerp_sweep.json").read_text())["rows"] if x["tag"]=="t080") |
| rep=json.loads((out/"benchmarks/alice_reproduction.json").read_text()) |
| plt.rcParams.update({'font.family':'DejaVu Sans','font.size':11,'axes.spines.top':False,'axes.spines.right':False,'axes.spines.left':False,'axes.titleweight':'bold','axes.labelcolor':'#475569','xtick.color':'#334155','ytick.color':'#475569','figure.facecolor':'white','axes.facecolor':'white','savefig.facecolor':'white'}) |
| colors=['#64748b','#2563eb','#0f9d8a'];fig,axs=plt.subplots(2,2,figsize=(13,8.6));fig.subplots_adjust(top=.84,bottom=.14,left=.07,right=.98,hspace=.55,wspace=.25) |
| fig.suptitle('BestTerm-440M vs. NextTerm-440M',x=.07,y=.97,ha='left',fontsize=23,fontweight='bold',color='#0f172a') |
| fig.text(.07,.918,'Stronger short-sequence completion; broadly preserved OEIS and polynomial accuracy.',fontsize=12,color='#475569') |
| for ax in axs.flat:ax.set_axisbelow(True);ax.grid(axis='y',color='#e2e8f0');ax.tick_params(axis='both',length=0) |
| ax=axs[0,0];x=np.arange(2) |
| for offset,vals,color,label in [(-.18,[30/57*100,32/57*100],colors[0],'NextTerm-440M'),(.18,[38/57*100,40/57*100],colors[1],'BestTerm-440M')]: |
| bars=ax.bar(x+offset,vals,.32,color=color,label=label);ax.bar_label(bars,labels=[f'{v:.2f}%' for v in vals],padding=4,fontsize=10) |
| ax.set_xticks(x,['Greedy','Beam-4']);ax.set_ylim(0,85);ax.set_title('Ryskina & Knight 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)');ax.legend(frameon=False,fontsize=9,loc='upper left',ncol=2) |
| labels=['Base','BestTerm\noriginal','BestTerm\nSpark'] |
| ax=axs[0,1];vals=[6555/19034*100,6532/19034*100,6536/19034*100];bars=ax.bar(range(3),vals,.6,color=colors);ax.bar_label(bars,labels=[f'{v:.3f}%' for v in vals],padding=4);ax.set_xticks(range(3),labels);ax.set_ylim(0,42);ax.set_title('OEIS-Eval-Neo 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)') |
| ax=axs[1,0];vals=[17.623926615,17.582547764,17.813680040];bars=ax.bar(range(3),vals,.6,color=colors);ax.bar_label(bars,labels=[f'{v:.4f}' for v in vals],padding=4);ax.set_xticks(range(3),labels);ax.set_ylim(0,22);ax.set_title('M1 Competition 111 路 lower is better',loc='left',pad=12);ax.set_ylabel('Macro MAPE');ax.axhline(17.798673617,color='#b45309',linestyle='--',linewidth=1.2);ax.text(2.4,20.1,'Naive2: 17.7987',ha='right',color='#92400e',fontsize=10) |
| ax=axs[1,1];tasks=['arithmetic','quadratic','cubic','quartic'];x=np.arange(4) |
| for offset,vals,color,label in [(-.24,[100*base[k]['accuracy'] for k in tasks],colors[0],'Base'),(0,[100*orig['poly_'+k] for k in tasks],colors[1],'BestTerm original'),(.24,[100*rep['poly_'+k] for k in tasks],colors[2],'BestTerm Spark')]:ax.bar(x+offset,vals,.22,color=color,label=label) |
| ax.set_xticks(x,['Arithmetic','Quadratic','Cubic','Quartic']);ax.set_ylim(0,110);ax.set_title('Polynomial continuation 路 higher is better',loc='left',pad=12);ax.set_ylabel('Exact next-term accuracy (%)');ax.legend(frameon=False,fontsize=8,ncol=3,loc='upper center') |
| fig.text(.07,.06,'Ryskina: matched Transformers comparison. Other panels: original experiment controls and saved Spark reproduction.\nHistorical July 2026 results; recovered September 2026. Selection used these benchmarks. See model card for protocols.',fontsize=10,color='#64748b',linespacing=1.6) |
| fig.savefig(out/'assets/benchmark_comparison.png',dpi=180);plt.close(fig) |
| print("Saved assets/benchmark_comparison.png", flush=True) |
|
|
