| """Eighth-tranche fixed evaluation (no training, no generation). FP32 + math SDPA, per-window arithmetic and bootstrap identical to evaluate_seventh.py. |
| Arms: CANDIDATE (eighth endpoint) and PARENT (step 15,106, seventh recovery-final.pt), both scored live. |
| Window gates: eighth_final (primary), seventh_final, sixth_final, final126 (no-regression). eighth_dev: diagnostic only (never gating). |
| Retention baselines A0/English93 and step3403/English599 are recorded per-unit rows (sha-pinned in FROZEN_GATES.json); candidate scored live. |
| Speed change vs seventh (time-boxed session): the two arms are scored concurrently on two GPUs (--device/--device2, one thread each); |
| each window is still one independent FP32 forward, so values do not depend on the split. Falls back to one device if only one is given.""" |
| import argparse, hashlib, json, math, os, random, sys, time |
| import torch |
| from torch.nn.attention import sdpa_kernel, SDPBackend |
| sys.path.insert(0,os.path.dirname(os.path.abspath(__file__))) |
| from frozen_stasis.config import ModelConfig |
| from frozen_stasis.model import StasisDecoder |
|
|
| def fsha(p): |
| h=hashlib.sha256(); f=open(p,'rb') |
| for b in iter(lambda:f.read(1<<24),b''): h.update(b) |
| return h.hexdigest() |
|
|
| def load_model(state,dev): |
| m=StasisDecoder(ModelConfig()); m.load_state_dict(state,strict=True); return m.to(dev).eval() |
|
|
| @torch.no_grad() |
| def nll_window(m,tokens,dev): |
| x=torch.tensor(tokens,dtype=torch.long,device=dev)[None] |
| with sdpa_kernel(SDPBackend.MATH): |
| logits,_=m(x[:,:-1]) |
| return float(torch.nn.functional.cross_entropy(logits.float().reshape(-1,logits.shape[-1]),x[0,1:],reduction='sum').double()) |
|
|
| @torch.no_grad() |
| def nll_english93(m,rows,dev): |
| per={} |
| for r in rows: |
| x=torch.tensor(r['input_ids'],dtype=torch.long,device=dev)[None] |
| with sdpa_kernel(SDPBackend.MATH): |
| logits,_=m(x) |
| l=torch.nn.functional.cross_entropy(logits.float()[0],torch.tensor(r['target_ids'],device=dev),reduction='none') |
| mask=torch.tensor(r['score_mask'],device=dev) |
| per[r['document_id']]=per.get(r['document_id'],0.0)+float(l[mask].double().sum()) |
| return per |
|
|
| def boot_delta(base,cand,seed=1337,draws=1000): |
| ids=sorted(base); rng=random.Random(seed); out=[] |
| for _ in range(draws): |
| s=rng.choices(ids,k=len(ids)); out.append(sum(cand[i][0]-base[i][0] for i in s)/sum(base[i][1] for i in s)) |
| return sorted(out)[int(0.95*draws)-1] |
|
|
| def boot_ratio(base,cand,seed=1337,draws=1000): |
| ids=sorted(base); rng=random.Random(seed); out=[] |
| for _ in range(draws): |
| s=rng.choices(ids,k=len(ids)); out.append(sum(cand[i] for i in s)/sum(base[i] for i in s)) |
| return sorted(out)[int(0.95*draws)-1] |
|
|
| def main(): |
| ap=argparse.ArgumentParser(); ap.add_argument('--gates',required=True); ap.add_argument('--candidate',required=True) |
| ap.add_argument('--parent',required=True); ap.add_argument('--english599',default=None); ap.add_argument('--out',required=True) |
| ap.add_argument('--device',default='cuda:0'); ap.add_argument('--device2',default=None); ap.add_argument('--limit',type=int,default=None); a=ap.parse_args() |
| import threading |
| t0=time.time(); g=json.load(open(a.gates)); base=os.path.dirname(os.path.abspath(a.gates)) |
| torch.backends.cuda.matmul.allow_tf32=False; torch.backends.cudnn.allow_tf32=False |
| for k,p in g['pinned_inputs'].items(): |
| if os.path.exists(os.path.join(base,p['path'])): assert fsha(os.path.join(base,p['path']))==p['sha256'],k |
| d0=torch.device(a.device); d1=torch.device(a.device2) if a.device2 else d0; L=a.limit |
| cstate=torch.load(a.candidate,map_location='cpu',weights_only=True)['model'] |
| cand0=load_model(cstate,d0); cand1=load_model(cstate,d1) if d1!=d0 else cand0; del cstate |
| par=load_model(torch.load(a.parent,map_location='cpu',weights_only=True)['model'],d1) |
| res={'gates_sha256':fsha(a.gates),'gates':{},'devices':[str(d0),str(d1)]}; rows={} |
| def windows(path): return json.load(open(os.path.join(base,path)))[:L] |
| PANELS=('eighth_final','seventh_final','sixth_final','final126','eighth_dev') |
| specs={p:(g[p] if p!='eighth_dev' else dict(g['dev_monitoring'],min_relative_ce_improvement=0.0,bootstrap_upper95_delta_strictly_below=0.0)) for p in PANELS} |
| W={p:windows(specs[p]['path']) for p in PANELS} |
| spec93=g['retention_english93']; ew=[json.loads(l) for l in open(os.path.join(base,spec93['windows_path']))] |
| if L: keep={r['document_id'] for r in ew[:L]}; ew=[r for r in ew if r['document_id'] in keep] |
| spec599=g['retention_english599'] |
| e599=json.load(open(a.english599))[:L] if (a.english599 and os.path.exists(a.english599) and fsha(a.english599)==spec599['windows_sha256']) else None |
| out={'cand':{},'par':{}}; errs=[] |
| def job0(): |
| try: |
| for p in PANELS: out['cand'][p]={w['component_id']:[nll_window(cand0,w['tokens'],d0),2048] for w in W[p]} |
| out['c93']=nll_english93(cand0,ew,d0) |
| except Exception as ex: errs.append(repr(ex)) |
| def job1(): |
| try: |
| for p in PANELS: out['par'][p]={w['component_id']:[nll_window(par,w['tokens'],d1),2048] for w in W[p]} |
| if e599 is not None: out['c599']={w['component_id']:nll_window(cand1,w['tokens'],d1) for w in e599} |
| except Exception as ex: errs.append(repr(ex)) |
| if d1!=d0: |
| th=[threading.Thread(target=job0),threading.Thread(target=job1)]; [x.start() for x in th]; [x.join() for x in th] |
| else: job0(); job1() |
| assert not errs, errs |
| for panel in PANELS: |
| spec=specs[panel]; bc=out['par'][panel]; cc=out['cand'][panel] |
| bce=sum(v[0] for v in bc.values())/sum(v[1] for v in bc.values()); cce=sum(v[0] for v in cc.values())/sum(v[1] for v in cc.values()) |
| imp=1-cce/bce; ub=boot_delta(bc,cc) |
| ok=(imp>=spec['min_relative_ce_improvement']) and ub<spec['bootstrap_upper95_delta_strictly_below'] |
| r_=dict(parent_ce=bce,candidate_ce=cce,relative_ce_improvement=imp,bootstrap_upper95_delta=ub,units=len(bc)) |
| if panel=='eighth_dev': res['diagnostics']={'eighth_dev':r_} |
| else: res['gates'][panel]=dict(r_,passed=ok) |
| rows[panel]={'parent':bc,'candidate':cc} |
| a0={(r:=json.loads(l))['id']:r['total_nll_nats'] for l in open(os.path.join(base,spec93['baseline_rows_path']))} |
| c93=out['c93']; b93={k:a0[k] for k in c93} |
| ub=boot_ratio(b93,c93); res['gates']['retention_english93']=dict(ratio=sum(c93.values())/sum(b93.values()),bootstrap_upper95_ratio=ub,units=len(c93),passed=ub<=spec93['max_upper95_ratio']) |
| rows['english93']=c93 |
| if e599 is not None: |
| b={(r:=json.loads(l))['component_id']:r['nll_nats'] for l in open(os.path.join(base,spec599['baseline_rows_path']))} |
| c=out['c599']; bb={k:b[k] for k in c} |
| ub=boot_ratio(bb,c); res['gates']['retention_english599']=dict(ratio=sum(c.values())/sum(bb.values()),bootstrap_upper95_ratio=ub,units=len(c),passed=ub<=spec599['max_upper95_ratio']) |
| rows['english599']=c |
| else: |
| res['gates']['retention_english599']=dict(passed=False,status='PANEL_NOT_MOUNTED_OR_SHA_MISMATCH') |
| res['all_pass']=all(v.get('passed') for v in res['gates'].values()) |
| res['decision']='RESEARCH_GAIN_ESTABLISHED_NO_AUTOMATIC_PROMOTION' if res['all_pass'] else 'NO_JOINT_RESEARCH_VALUE_ESTABLISHED' |
| res['limit']=L; res['elapsed_seconds']=round(time.time()-t0,1) |
| os.makedirs(a.out,exist_ok=True) |
| json.dump(res,open(os.path.join(a.out,'EVAL_RESULT.json'),'w'),indent=1); json.dump(rows,open(os.path.join(a.out,'EVAL_ROWS.json'),'w')) |
| print(json.dumps(res,indent=1)) |
| if __name__=='__main__': main() |
|
|
