Download reproduction/evaluate_objective.py from nohi191212/ModelCollaboration: direct link, hf CLI and curl.
- Browser
- Download file 4.22 kB
-
https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/reproduction/evaluate_objective.py
- Command line
-
hf download hf://nohi191212/ModelCollaboration/reproduction/evaluate_objective.py
-
curl -L -o evaluate_objective.py https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/reproduction/evaluate_objective.py
4.22 kB
| """Evaluate fixed checkpoints using validation-selected thresholds. | |
| Algorithm copied from the executed five-seed evaluator. Only path/device | |
| arguments and the main guard differ from the executed script. | |
| """ | |
| from pathlib import Path | |
| import argparse | |
| import json | |
| import numpy as np | |
| def main(): | |
| p=argparse.ArgumentParser() | |
| p.add_argument('--root',type=Path,required=True,help='Prepared project with outputs/router_full_20260910 caches') | |
| p.add_argument('--run',type=Path,required=True) | |
| p.add_argument('--device',default='cuda') | |
| a=p.parse_args() | |
| import torch | |
| from routing import train_router_formal as formal | |
| formal.ROOT=a.root.resolve() | |
| formal.WORK=formal.ROOT/'outputs/router_full_20260910' | |
| cfg=json.loads((a.run/'config.json').read_text()) | |
| torch.set_num_threads(cfg['cpu_threads']) | |
| device=torch.device(a.device) | |
| records={};logits={} | |
| model=None | |
| for split in ['val','test']: | |
| rows,data=formal.load_data(cfg,split,device);records[split]=rows | |
| for key in ['hidden','confidence']: | |
| norm=np.load(a.run/(key+'_normalization.npz')) | |
| data[key]=(data[key]-torch.from_numpy(norm['mean']).to(device))/torch.from_numpy(norm['std']).to(device) | |
| if model is None: | |
| model=formal.FeatureRouter(cfg,data['hidden'].shape[1],data['confidence'].shape[1],data['output'].shape[1]).to(device) | |
| model.load_state_dict(torch.load(a.run/'best.pt',map_location=device,weights_only=True)['state_dict']);model.eval() | |
| with torch.inference_mode(): | |
| logits[split]=torch.cat([model(data,torch.arange(i,min(i+8192,len(rows)),device=device)).cpu() for i in range(0,len(rows),8192)]).numpy() | |
| del data | |
| saved_validation=np.load(a.run/'validation_logits.npy') | |
| assert np.allclose(logits['val'],saved_validation,rtol=1e-5,atol=1e-6), 'Validation logits changed after checkpoint reload' | |
| logits['val']=saved_validation | |
| scores={} | |
| for split,raw in logits.items(): | |
| if cfg['target']=='four_state': | |
| prob=torch.from_numpy(raw).softmax(1).numpy() | |
| scores[split]={'log_ratio':raw[:,1]-raw[:,2],'probability_difference':prob[:,1]-prob[:,2]} | |
| else:scores[split]={cfg['ranking']:torch.from_numpy(raw[:,0]).sigmoid().numpy() if cfg['target'] in ['error','route'] else raw[:,0]} | |
| np.savez(a.run/'paired_logits.npz',validation=logits['val'],test=logits['test']) | |
| result={'config':cfg,'test_rows':len(records['test']),'validation_rows':len(records['val']),'comparisons':{}} | |
| for ranking in scores['val']: | |
| val,test=scores['val'][ranking],scores['test'][ranking] | |
| selection_score=formal.ranked_curve(records['val'],cfg['endpoint'],val)['selection_score'] | |
| ranked_test=formal.ranked_curve(records['test'],cfg['endpoint'],test) | |
| formal.BUDGETS=[i/100 for i in range(101)] | |
| thresholds=formal.validation_thresholds(records['val'],cfg['endpoint'],val) | |
| vc=formal.fixed_curve(records['val'],cfg['endpoint'],val,thresholds)['curve'] | |
| tc=formal.fixed_curve(records['test'],cfg['endpoint'],test,thresholds)['curve'] | |
| best=min(range(1,100),key=lambda i:(-vc[i]['metric'],vc[i]['actual_fraction'],vc[i]['budget'])) | |
| fixed25=sum((tc[i]['metric']+tc[i+5]['metric'])*.5*.05 for i in range(0,25,5))/.25 | |
| result['comparisons'][ranking]={'validation_score25':selection_score,'test_ranked_score25':ranked_test['selection_score'], | |
| 'test_exact_budget_diagnostic':ranked_test['curve'],'validation_curve':vc,'test_curve':tc, | |
| 'validation_selected':vc[best],'test_selected':tc[best],'test_fixed_score25':fixed25, | |
| 'checkpoint_ranking':cfg['ranking'],'same_checkpoint_control':ranking!=cfg['ranking']} | |
| formal.BUDGETS=[0,.05,.10,.15,.20,.25,.50,.75,1.] | |
| result['small_test']=formal.metric(records['test'],cfg['endpoint'],np.zeros(len(records['test']),dtype=bool)) | |
| result['large_test']=formal.metric(records['test'],cfg['endpoint'],np.ones(len(records['test']),dtype=bool)) | |
| (a.run/'test_evaluation.json').write_text(json.dumps(result,indent=2)) | |
| print(json.dumps({'run':cfg['id'],'rankings':list(result['comparisons'])}),flush=True) | |
| if __name__=='__main__':main() | |