| import os |
| os.environ.setdefault('MPLCONFIGDIR',os.path.abspath('.cache/matplotlib')) |
| import json,hashlib,joblib,numpy as np,torch |
| from pathlib import Path |
| from chisco import extract,metrics |
| from scipy.stats import binomtest |
|
|
| def evaluate(): |
| manifest=json.load(open('reports/frozen_selection.json'));b=extract('test',evaluation=True);a=extract('train');v=extract('validation');known=set(a['text'])|set(v['text']);keep=np.array([t not in known for t in b['text']]);results=[] |
| for kind in ['x','eog']: |
| path=f'checkpoints/chisco_sub01_{kind}_blocksplit.joblib';assert hashlib.sha256(Path(path).read_bytes()).hexdigest()==manifest['chisco_checkpoints'][path] |
| m=joblib.load(path);p=m.predict(b[kind]);result={'model':kind,'test_n':len(p),'classes':39,'chance':1/39,'metrics':metrics(b['y'],p),'text_disjoint_n':int(keep.sum()),'text_disjoint_metrics':metrics(b['y'][keep],p[keep]) if keep.any() else None,'class_counts':np.bincount(b['y'],minlength=39).tolist()};results.append(result);np.savez(f'reports/predictions/chisco_{kind}.npz',y=b['y'],pred=p,block=b['block'],disjoint=keep) |
| path='checkpoints/chisco_sub01_eegnet_blocksplit.pt' |
| if path in manifest['chisco_checkpoints']: |
| from chisco_neural import data |
| from braindecode.models import EEGNet |
| assert hashlib.sha256(Path(path).read_bytes()).hexdigest()==manifest['chisco_checkpoints'][path] |
| r=torch.load(path,map_location='cpu',weights_only=False);d=data('test',evaluation=True);assert np.array_equal(d['y'],b['y']);torch.set_num_threads(2);m=EEGNet(n_chans=122,n_outputs=39,n_times=600,drop_prob=.5);m.load_state_dict(r['model']);m.eval() |
| with torch.inference_mode():p=torch.cat([m(q) for q in torch.tensor(np.clip(d['x']/r['scale'],-20,20)).split(64)]).argmax(1).numpy() |
| results.append({'model':'eegnet','test_n':len(p),'classes':39,'chance':1/39,'metrics':metrics(b['y'],p),'text_disjoint_n':int(keep.sum()),'text_disjoint_metrics':metrics(b['y'][keep],p[keep]) if keep.any() else None});np.savez('reports/predictions/chisco_eegnet.npz',y=b['y'],pred=p,block=b['block'],disjoint=keep) |
| path='checkpoints/chisco_sub01_cbramod_blocksplit.joblib' |
| if path in manifest['chisco_checkpoints']: |
| from chisco_cbramod import extract as extract_cbramod |
| assert hashlib.sha256(Path(path).read_bytes()).hexdigest()==manifest['chisco_checkpoints'][path] |
| state=joblib.load(path);f,d=extract_cbramod('test',state['groups'],evaluation=True);assert np.array_equal(d['y'],b['y']);pred=state['model'].predict(f);np.savez('reports/predictions/chisco_cbramod.npz',y=b['y'],pred=pred,block=b['block'],disjoint=keep);results.append({'model':'cbramod','test_n':len(pred),'classes':39,'chance':1/39,'metrics':metrics(b['y'],pred),'text_disjoint_n':int(keep.sum()),'text_disjoint_metrics':metrics(b['y'][keep],pred[keep]) if keep.any() else None}) |
| majority=int(np.bincount(a['y'],minlength=39).argmax());pred=np.full(len(b['y']),majority);np.savez('reports/predictions/chisco_majority_prior.npz',y=b['y'],pred=pred,block=b['block'],disjoint=keep) |
| results.append({'model':'majority_prior','test_n':len(pred),'classes':39,'chance':1/39,'metrics':metrics(b['y'],pred),'text_disjoint_n':int(keep.sum()),'text_disjoint_metrics':metrics(b['y'][keep],pred[keep]) if keep.any() else None,'majority_class_fitted_on_train':majority}) |
| for r in results: |
| z=np.load(f"reports/predictions/chisco_{r['model']}.npz");blocks=np.unique(z['block']);rng=np.random.default_rng(2026);scores=[] |
| for _ in range(2000): |
| ids=np.concatenate([np.flatnonzero(z['block']==g) for g in rng.choice(blocks,len(blocks),replace=True)]);scores.append(float(np.mean(z['y'][ids]==z['pred'][ids]))) |
| r['block_bootstrap_accuracy_ci95']=np.quantile(scores,[.025,.975]).tolist() |
| json.dump(results,open('reports/chisco_test.json','w'),indent=2);print([(r['model'],r['metrics']['accuracy']) for r in results]) |
| if __name__=='__main__': |
| evaluate() |
| from chisco_five import evaluate as evaluate_five |
| evaluate_five() |
|
|
