scripts/benchmark.py

import os,time,json,collections
os.environ.setdefault('MPLCONFIGDIR',os.path.abspath('.cache/matplotlib'))
from pathlib import Path
import numpy as np,joblib
from inference import Predictor
from bci_data import cached
from evaluate import candidates

def run():
 groups={}
 for s,r,p in candidates():
  if s!=1:continue
  groups[(r['model'],r['montage'],r['channels'],r.get('seed','deterministic'))]=(r,p)
 x,_,_=cached(1,'validation');out=[]
 for key,(r,p) in sorted(groups.items(),key=lambda x:str(x[0])):
  t=time.perf_counter();m=Predictor(p);loadtime=time.perf_counter()-t;m.predict(x[0]);times=[m.predict(x[i])['latency_ms'] for i in range(5)];extra={}
  if m.neural:extra['parameters']=sum(q.numel() for q in m.model.parameters())
  elif hasattr(m.model,'steps'):
   last=m.model.steps[-1][1]
   if hasattr(last,'n_support_'):extra['support_vectors']=int(last.n_support_.sum());extra['feature_dimensions']=last.n_features_in_
   elif hasattr(last,'coef_'):extra['coefficients']=int(last.coef_.size)
  out.append({'model':key[0],'montage':key[1],'channels':key[2],'seed':key[3],'representative_subject':1,'source_split':'validation','warm_trials':5,'median_preprocess_and_inference_ms':float(np.median(times)),'range_ms':[min(times),max(times)],'load_seconds':loadtime,'checkpoint_bytes':p.stat().st_size,'device':'cpu',**extra})
 Path('reports/inference_benchmarks.json').write_text(json.dumps(out,indent=2));print('Benchmarked',len(out),'configurations')
if __name__=='__main__':run()