ModelCollaboration / reproduction /evaluate_objective.py
nohi191212's picture
Publish five-seed probability-difference results, 400 routers, and revision 32
466c2b6 verified
Raw History Blame Contribute Delete
4.22 kB
"""Evaluate fixed checkpoints using validation-selected thresholds.
Algorithm copied from the executed five-seed evaluator. Only path/device
arguments and the main guard differ from the executed script.
"""
from pathlib import Path
import argparse
import json
import numpy as np
def main():
p=argparse.ArgumentParser()
p.add_argument('--root',type=Path,required=True,help='Prepared project with outputs/router_full_20260910 caches')
p.add_argument('--run',type=Path,required=True)
p.add_argument('--device',default='cuda')
a=p.parse_args()
import torch
from routing import train_router_formal as formal
formal.ROOT=a.root.resolve()
formal.WORK=formal.ROOT/'outputs/router_full_20260910'
cfg=json.loads((a.run/'config.json').read_text())
torch.set_num_threads(cfg['cpu_threads'])
device=torch.device(a.device)
records={};logits={}
model=None
for split in ['val','test']:
rows,data=formal.load_data(cfg,split,device);records[split]=rows
for key in ['hidden','confidence']:
norm=np.load(a.run/(key+'_normalization.npz'))
data[key]=(data[key]-torch.from_numpy(norm['mean']).to(device))/torch.from_numpy(norm['std']).to(device)
if model is None:
model=formal.FeatureRouter(cfg,data['hidden'].shape[1],data['confidence'].shape[1],data['output'].shape[1]).to(device)
model.load_state_dict(torch.load(a.run/'best.pt',map_location=device,weights_only=True)['state_dict']);model.eval()
with torch.inference_mode():
logits[split]=torch.cat([model(data,torch.arange(i,min(i+8192,len(rows)),device=device)).cpu() for i in range(0,len(rows),8192)]).numpy()
del data
saved_validation=np.load(a.run/'validation_logits.npy')
assert np.allclose(logits['val'],saved_validation,rtol=1e-5,atol=1e-6), 'Validation logits changed after checkpoint reload'
logits['val']=saved_validation
scores={}
for split,raw in logits.items():
if cfg['target']=='four_state':
prob=torch.from_numpy(raw).softmax(1).numpy()
scores[split]={'log_ratio':raw[:,1]-raw[:,2],'probability_difference':prob[:,1]-prob[:,2]}
else:scores[split]={cfg['ranking']:torch.from_numpy(raw[:,0]).sigmoid().numpy() if cfg['target'] in ['error','route'] else raw[:,0]}
np.savez(a.run/'paired_logits.npz',validation=logits['val'],test=logits['test'])
result={'config':cfg,'test_rows':len(records['test']),'validation_rows':len(records['val']),'comparisons':{}}
for ranking in scores['val']:
val,test=scores['val'][ranking],scores['test'][ranking]
selection_score=formal.ranked_curve(records['val'],cfg['endpoint'],val)['selection_score']
ranked_test=formal.ranked_curve(records['test'],cfg['endpoint'],test)
formal.BUDGETS=[i/100 for i in range(101)]
thresholds=formal.validation_thresholds(records['val'],cfg['endpoint'],val)
vc=formal.fixed_curve(records['val'],cfg['endpoint'],val,thresholds)['curve']
tc=formal.fixed_curve(records['test'],cfg['endpoint'],test,thresholds)['curve']
best=min(range(1,100),key=lambda i:(-vc[i]['metric'],vc[i]['actual_fraction'],vc[i]['budget']))
fixed25=sum((tc[i]['metric']+tc[i+5]['metric'])*.5*.05 for i in range(0,25,5))/.25
result['comparisons'][ranking]={'validation_score25':selection_score,'test_ranked_score25':ranked_test['selection_score'],
'test_exact_budget_diagnostic':ranked_test['curve'],'validation_curve':vc,'test_curve':tc,
'validation_selected':vc[best],'test_selected':tc[best],'test_fixed_score25':fixed25,
'checkpoint_ranking':cfg['ranking'],'same_checkpoint_control':ranking!=cfg['ranking']}
formal.BUDGETS=[0,.05,.10,.15,.20,.25,.50,.75,1.]
result['small_test']=formal.metric(records['test'],cfg['endpoint'],np.zeros(len(records['test']),dtype=bool))
result['large_test']=formal.metric(records['test'],cfg['endpoint'],np.ones(len(records['test']),dtype=bool))
(a.run/'test_evaluation.json').write_text(json.dumps(result,indent=2))
print(json.dumps({'run':cfg['id'],'rankings':list(result['comparisons'])}),flush=True)
if __name__=='__main__':main()