scripts/foundation_gpu.py

"""CUDA-only foundation adaptation. Official train/validation only; resumable epochs."""
import os,json,time,argparse,random
from pathlib import Path
from recovery_state import completed, atomic_json
import numpy as np,torch,joblib
from torch import nn
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from bci_data import cached,preprocess,montage
from classical import metrics
from foundation_models import Foundation,FoundationClassifier
from gpu_utils import require_cuda,expired,atomic_save,cpu_state,install_signals

def scale_for(kind):return 1000. if kind=='reve' else 100.
def tensors(subject,n,kind):
 x,y,names=cached(subject,'train');v,z,_=cached(subject,'validation');idx=montage(names,n);scale=scale_for(kind)
 return preprocess(x,idx)/scale,y,preprocess(v,idx)/scale,z,idx,names

def probe(kind,n,s):
 name=f'strict_{kind}_probe_distributed_{n}_sub-{s:02d}';out=Path('experiments')/(name+'.json')
 if completed(name):return
 device=require_cuda();x,y,v,z,idx,names=tensors(s,n,kind);encoder=Foundation(kind,[names[i] for i in idx]).to(device).eval();features=[];start=time.time()
 for split,a in [('train',x),('validation',v)]:
  path=Path('preprocessing')/f'{name}_{split}_features.npy'
  if path.exists():features.append(np.load(path));continue
  chunks=[]
  with torch.inference_mode():
   for q in torch.tensor(a).split(16):
    if expired():return
    with torch.autocast('cuda',dtype=torch.bfloat16):f=encoder(q.to(device))
    chunks.append(f.float().cpu().numpy())
  f=np.concatenate(chunks);np.save(path,f);features.append(f)
 del encoder;torch.cuda.empty_cache();best=None;search=[]
 for c in [.001,.01,.1,1.]:
  if expired():return
  model=make_pipeline(StandardScaler(),LogisticRegression(C=c,max_iter=1500,class_weight='balanced'));model.fit(features[0],y);m=metrics(z,model.predict(features[1]));search.append({'C':c,**m})
  if best is None or m['balanced_accuracy']>best[0]:best=(m['balanced_accuracy'],model,c,m)
 Path('reports/validation_predictions').mkdir(exist_ok=True)
 np.savez_compressed(f'reports/validation_predictions/{name}.npz',y=z,pred=best[1].predict(features[1]))
 joblib.dump({'model':best[1],'kind':kind,'indices':idx,'names':names,'subject':s,'scale':scale_for(kind),'preprocessing':'car_1-40_imagined0to2s_200hz','feature_precision':'cuda_bfloat16'},f'checkpoints/{name}.joblib')
 atomic_json(out, {'name':name,'model':kind+'_probe','subject':s,'channels':n,'montage':'distributed','split':'validation','selected_C':best[2],'metrics':best[3],'search':search,'seconds':time.time()-start,'device':'cuda','precision':'bfloat16'});print(name,best[3],flush=True)

def finetune(kind,mode,n,s,epochs=50,lr=.0001,seed=2026):
 name=f'strict_{kind}_{mode}_distributed_{n}_sub-{s:02d}_seed-{seed}';out=Path('experiments')/(name+'.json')
 if completed(name):return
 device=require_cuda();random.seed(seed);np.random.seed(seed);torch.manual_seed(seed);torch.cuda.manual_seed_all(seed);starttime=time.time()
 x,y,v,z,idx,names=tensors(s,n,kind);x=np.clip(x,-20,20);v=np.clip(v,-20,20);model=FoundationClassifier(kind,[names[i] for i in idx],mode).to(device)
 opt=torch.optim.AdamW([{'params':[p for p in model.encoder.parameters() if p.requires_grad],'lr':lr/10},{'params':model.head.parameters(),'lr':lr}],weight_decay=.01)
 loader=torch.utils.data.DataLoader(torch.utils.data.TensorDataset(torch.tensor(x),torch.tensor(y)),batch_size=16,shuffle=True);vx=torch.tensor(v,device=device)
 best=-1;best_epoch=-1;start=0;history=[];last=Path(f'checkpoints/{name}_last.pt');bestpath=Path(f'checkpoints/{name}.pt')
 if last.exists():
  state=torch.load(last,map_location='cpu',weights_only=False);model.load_state_dict(state['model']);opt.load_state_dict(state['optimizer']);start=state['epoch']+1;best=state['best'];best_epoch=state['best_epoch'];history=state['history'];torch.set_rng_state(state['rng']);torch.cuda.set_rng_state_all(state['cuda_rng']);random.setstate(state['python_rng']);np.random.set_state(state['numpy_rng'])
 criterion=nn.CrossEntropyLoss(label_smoothing=.05)

 if not last.exists() and bestpath.exists():
  state=torch.load(bestpath,map_location='cpu',weights_only=False);model.load_state_dict(state['model']);best=state['validation_metrics']['balanced_accuracy'];best_epoch=state['epoch'];start=best_epoch+1
  print(name,'Resuming best weights only: optimizer/RNG snapshot unavailable',flush=True)
 if last.exists() and start-1-best_epoch>=12:start=epochs
 for epoch in range(start,epochs):
  if expired():return
  model.train();losses=[]
  for a,b in loader:
   a=a.to(device);b=b.to(device);a=a*(.9+.2*torch.rand((len(a),1,1),device=device))+torch.randn_like(a)*(.02 if kind=='csbrain' else .002)
   opt.zero_grad(set_to_none=True)
   with torch.autocast('cuda',dtype=torch.bfloat16):logits=model(a);loss=criterion(logits,b)
   if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss')
   loss.backward();nn.utils.clip_grad_norm_(model.parameters(),1.);opt.step();losses.append(loss.item())
  model.eval()
  with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):pred=torch.cat([model(q) for q in vx.split(16)]).float().argmax(1).cpu().numpy()
  m=metrics(z,pred);history.append({'epoch':epoch,'loss':float(np.mean(losses)),**m});score=m['balanced_accuracy']
  if score>best:
   best=score;best_epoch=epoch;atomic_save({'model':cpu_state(model),'kind':kind+'_'+mode,'channels':n,'indices':idx,'names':names,'scale':np.full((1,n,1),scale_for(kind),dtype=np.float32),'seed':seed,'subjects':[s],'epoch':epoch,'validation_metrics':m,'training_device':'cuda','precision':'bfloat16'},bestpath)
  atomic_save({'model':cpu_state(model),'optimizer':opt.state_dict(),'epoch':epoch,'best':best,'best_epoch':best_epoch,'history':history,'rng':torch.get_rng_state(),'cuda_rng':torch.cuda.get_rng_state_all(),'python_rng':random.getstate(),'numpy_rng':np.random.get_state()},last)
  print(name,epoch,score,float(np.mean(losses)),flush=True)
  if epoch-best_epoch>=12:break
 if expired():return
 beststate=torch.load(bestpath,map_location='cpu',weights_only=False)
 model.load_state_dict(beststate['model']);model.eval()
 with torch.inference_mode(),torch.autocast('cuda',dtype=torch.bfloat16):pred=torch.cat([model(q) for q in vx.split(16)]).float().argmax(1).cpu().numpy()
 Path('reports/validation_predictions').mkdir(exist_ok=True)
 np.savez_compressed(f'reports/validation_predictions/{name}.npz',y=z,pred=pred)
 atomic_json(out, {'name':name,'model':kind+'_'+mode,'channels':n,'montage':'distributed','subjects':[s],'seed':seed,'split':'validation','best_epoch':best_epoch,'metrics':beststate['validation_metrics'],'history':history,'seconds':time.time()-starttime,'device':'cuda','precision':'bfloat16','lr':lr,'parameters':sum(p.numel() for p in model.parameters())})

if __name__=='__main__':
 from experiment_guard import assert_tuning_open
 assert_tuning_open();require_cuda();install_signals();torch.set_num_threads(4)
 p=argparse.ArgumentParser();p.add_argument('--model',choices=['reve','csbrain'],required=True);p.add_argument('--mode',choices=['probe','partial','full'],default='probe');p.add_argument('--channels',nargs='+',type=int,default=[64,32,16,8]);p.add_argument('--subjects',nargs='+',type=int,default=list(range(1,16)));p.add_argument('--epochs',type=int,default=50);a=p.parse_args()
 for n in a.channels:
  for s in a.subjects:
   if expired():raise SystemExit(3)
   probe(a.model,n,s) if a.mode=='probe' else finetune(a.model,a.mode,n,s,a.epochs)