"""Seventh-tranche fixed evaluation (no training, no generation). FP32 + math SDPA, identical arithmetic to evaluate_sixth.py. Arms: CANDIDATE (seventh endpoint) and PARENT (step 12,269, from the sixth recovery-final.pt), both scored live. Window gates: seventh_final (primary), sixth_final, final126 (no-regression). seventh_dev: diagnostic only (never gating). Retention baselines A0/English93 and step3403/English599 are the recorded per-unit rows from the fifth T4 FP32 evaluation (sha-pinned in FROZEN_GATES.json); the candidate is scored live with the same arithmetic.""" import argparse, hashlib, json, math, os, random, sys, time import torch from torch.nn.attention import sdpa_kernel, SDPBackend sys.path.insert(0,os.path.dirname(os.path.abspath(__file__))) from frozen_stasis.config import ModelConfig from frozen_stasis.model import StasisDecoder def fsha(p): h=hashlib.sha256(); f=open(p,'rb') for b in iter(lambda:f.read(1<<24),b''): h.update(b) return h.hexdigest() def load_model(state,dev): m=StasisDecoder(ModelConfig()); m.load_state_dict(state,strict=True); return m.to(dev).eval() @torch.no_grad() def nll_window(m,tokens,dev): x=torch.tensor(tokens,dtype=torch.long,device=dev)[None] with sdpa_kernel(SDPBackend.MATH): logits,_=m(x[:,:-1]) return float(torch.nn.functional.cross_entropy(logits.float().reshape(-1,logits.shape[-1]),x[0,1:],reduction='sum').double()) @torch.no_grad() def nll_english93(m,rows,dev): per={} for r in rows: x=torch.tensor(r['input_ids'],dtype=torch.long,device=dev)[None] with sdpa_kernel(SDPBackend.MATH): logits,_=m(x) l=torch.nn.functional.cross_entropy(logits.float()[0],torch.tensor(r['target_ids'],device=dev),reduction='none') mask=torch.tensor(r['score_mask'],device=dev) per[r['document_id']]=per.get(r['document_id'],0.0)+float(l[mask].double().sum()) return per def boot_delta(base,cand,seed=1337,draws=1000): ids=sorted(base); rng=random.Random(seed); out=[] for _ in range(draws): s=rng.choices(ids,k=len(ids)); out.append(sum(cand[i][0]-base[i][0] for i in s)/sum(base[i][1] for i in s)) return sorted(out)[int(0.95*draws)-1] def boot_ratio(base,cand,seed=1337,draws=1000): ids=sorted(base); rng=random.Random(seed); out=[] for _ in range(draws): s=rng.choices(ids,k=len(ids)); out.append(sum(cand[i] for i in s)/sum(base[i] for i in s)) return sorted(out)[int(0.95*draws)-1] def main(): ap=argparse.ArgumentParser(); ap.add_argument('--gates',required=True); ap.add_argument('--candidate',required=True) ap.add_argument('--parent',required=True); ap.add_argument('--english599',default=None); ap.add_argument('--out',required=True) ap.add_argument('--device',default='cuda:0'); ap.add_argument('--limit',type=int,default=None); a=ap.parse_args() t0=time.time(); g=json.load(open(a.gates)); base=os.path.dirname(os.path.abspath(a.gates)) torch.backends.cuda.matmul.allow_tf32=False; torch.backends.cudnn.allow_tf32=False for k,p in g['pinned_inputs'].items(): if os.path.exists(os.path.join(base,p['path'])): assert fsha(os.path.join(base,p['path']))==p['sha256'],k dev=torch.device(a.device); L=a.limit cand=load_model(torch.load(a.candidate,map_location='cpu',weights_only=True)['model'],dev) par=load_model(torch.load(a.parent,map_location='cpu',weights_only=True)['model'],dev) res={'gates_sha256':fsha(a.gates),'gates':{}}; rows={} def windows(path): return json.load(open(os.path.join(base,path)))[:L] for panel in ('seventh_final','sixth_final','final126','seventh_dev'): spec=g[panel] if panel!='seventh_dev' else dict(g['dev_monitoring'],min_relative_ce_improvement=0.0,bootstrap_upper95_delta_strictly_below=0.0) ws=windows(spec['path']); bc={};cc={} for w in ws: bc[w['component_id']]=[nll_window(par,w['tokens'],dev),2048]; cc[w['component_id']]=[nll_window(cand,w['tokens'],dev),2048] bce=sum(v[0] for v in bc.values())/sum(v[1] for v in bc.values()); cce=sum(v[0] for v in cc.values())/sum(v[1] for v in cc.values()) imp=1-cce/bce; ub=boot_delta(bc,cc) ok=(imp>=spec['min_relative_ce_improvement']) and ub