scripts/chisco_five.py

"""Predefined constrained semantic subset: five most frequent TRAIN categories, not best EEG scores."""
import json,time,joblib,numpy as np
from pathlib import Path
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score,balanced_accuracy_score,f1_score,confusion_matrix
from chisco import extract

def met(y,p,classes):return {'accuracy':accuracy_score(y,p),'balanced_accuracy':balanced_accuracy_score(y,p),'macro_f1':f1_score(y,p,labels=classes,average='macro',zero_division=0),'confusion_matrix':confusion_matrix(y,p,labels=classes).tolist()}
def train():
 a=extract('train');b=extract('validation');keep=~np.isin(a['text'],b['text']);counts=np.bincount(a['y'][keep],minlength=39);classes=np.sort(np.argsort(counts,kind='stable')[-5:]);ia=keep&np.isin(a['y'],classes);ib=np.isin(b['y'],classes);names=json.load(open('datasets/chisco_metadata/json/classnumber.json'))
 for kind in ['x','eog','cbramod']:
  name=f'chisco_sub01_five_{kind}_blocksplit';out=Path('experiments')/(name+'.json')
  if out.exists():continue
  x=np.load('preprocessing/chisco_cbramod_train.npy') if kind=='cbramod' else a[kind];v=np.load('preprocessing/chisco_cbramod_validation.npy') if kind=='cbramod' else b[kind];best=None
  for c in [.001,.01,.1,1]:
   model=make_pipeline(StandardScaler(),LogisticRegression(C=c,max_iter=1500,class_weight='balanced'));model.fit(x[ia],a['y'][ia]);m=met(b['y'][ib],model.predict(v[ib]),classes)
   if best is None or m['balanced_accuracy']>best[0]:best=(m['balanced_accuracy'],model,c,m)
  joblib.dump({'model':best[1],'classes':classes,'majority_class':int(classes[np.argmax(counts[classes])])},f'checkpoints/{name}.joblib');out.write_text(json.dumps({'name':name,'split':'validation','task':'five most frequent training semantic classes','class_ids':classes.tolist(),'class_names':[names[str(c)] for c in classes],'train_n':int(ia.sum()),'validation_n':int(ib.sum()),'chance_balanced_accuracy':.2,'C':best[2],'metrics':best[3]},ensure_ascii=False,indent=2));print(name,best[3],flush=True)
def evaluate():
 manifest=json.load(open('reports/frozen_selection.json'));b=extract('test',evaluation=True);out=[]
 for kind in ['x','eog','cbramod']:
  path=f'checkpoints/chisco_sub01_five_{kind}_blocksplit.joblib'
  if path not in manifest['chisco_checkpoints']:continue
  import hashlib
  assert hashlib.sha256(Path(path).read_bytes()).hexdigest()==manifest['chisco_checkpoints'][path]
  r=joblib.load(path);classes=r['classes'];mask=np.isin(b['y'],classes)
  if not mask.any():
   out.append({'model':kind,'classes':classes.tolist(),'test_n':0,'status':'No test trials in the training-selected vocabulary'});continue
  x=np.load('preprocessing/chisco_cbramod_test.npy') if kind=='cbramod' else b[kind];p=r['model'].predict(x[mask]);out.append({'model':kind,'classes':classes.tolist(),'test_n':int(mask.sum()),'test_class_counts':{str(c):int(np.sum(b['y'][mask]==c)) for c in classes},'observed_test_classes':np.unique(b['y'][mask]).tolist(),'chance_balanced_accuracy':.2,'metrics':met(b['y'][mask],p,classes),'majority_prior_metrics':met(b['y'][mask],np.full(mask.sum(),r['majority_class']),classes)})
  blocks=b['block'][mask];groups=np.unique(blocks);rng=np.random.default_rng(2026);correct=p==b['y'][mask];scores=[]
  for _ in range(2000):
   ids=np.concatenate([np.flatnonzero(blocks==g) for g in rng.choice(groups,len(groups),replace=True)]);scores.append(float(correct[ids].mean()))
  out[-1]['block_bootstrap_accuracy_ci95']=np.quantile(scores,[.025,.975]).tolist();out[-1]['test_category_coverage']=len(np.unique(b['y'][mask]));np.savez(f'reports/predictions/chisco_five_{kind}.npz',y=b['y'][mask],pred=p,block=blocks)
 Path('reports/chisco_five_test.json').write_text(json.dumps(out,indent=2));print('Constrained five-category test:',out,flush=True)
if __name__=='__main__':
 from experiment_guard import assert_tuning_open
 assert_tuning_open();train()