cuerdas / 013 /iterate.py
Eriksanchezolvera's picture
Cuerdas 013-018
5851d4e verified
Raw History Blame Contribute Delete
4.34 kB
# Iteraciones 013c+ : mismos splits que 013b. test dividido en test_dev (iteraciones) y test_fresh (solo ganador).
import sys; exec(open('common.py').read())
FMT="natural_labeled"; pos=lambda m:len(ids(" "+FORMATS[FMT](m)))
random.seed(7); ct=cov_test[:]; ut=unc_test[:]; random.shuffle(ct); random.shuffle(ut)
dev=ct[:112]+ut[:268]; fresh=ct[112:]+ut[268:]
cal_all=cov_cal+unc_cal
_,TL=eval_text(FMT,cov_train,keep_logits=True)
okT_cal,_=eval_text(FMT,cal_all); okT_dev,_=eval_text(FMT,dev)
class Fact(nn.Module): # una ranura por campo, cada ranura depende SOLO de su campo (binding composicional por posición)
def __init__(s,mode):
super().__init__(); s.mode=mode; s.K=4 if mode=="fact4" else 1
s.tab=nn.ParameterList([nn.Parameter(E(ids(" "+L[0])[:1]).detach().repeat(len(L),1)+0.01*torch.randn(len(L),D)) for L in FL])
with torch.no_grad(): # init: embedding de la palabra del valor
for f,L in enumerate(FL):
for j,w in enumerate(L): s.tab[f][j]=E(ids(" "+w))[0]
s.role=nn.Parameter(torch.zeros(4,D))
def forward(s,mb):
v=torch.stack([s.tab[f][mb[:,f]]+s.role[f] for f in range(4)],1)
return v if s.mode=="fact4" else v.sum(1,keepdim=True)
class MLP(nn.Module):
def __init__(s,K): super().__init__(); s.a=Adapter(K); s.K=K
def forward(s,mb): return s.a(mb)
def train(ad,steps,distill=True,Tk=2.0,bs=32):
opt=torch.optim.AdamW(ad.parameters(),3e-3)
for st_ in range(steps):
b=random.sample(cov_train,bs); mb=torch.tensor(b); f=st_%4
v=ad(mb); B=len(b)
lg=lm(inputs_embeds=torch.cat([E(PRE).expand(B,-1,-1),v,E(Q[f]).expand(B,-1,-1)],1)).logits[:,-1][:,cand[f]]
loss=F.cross_entropy(lg,mb[:,f])
if distill: loss=loss+Tk*Tk*F.kl_div(F.log_softmax(lg/Tk,1),F.softmax(torch.stack([TL[m][f] for m in b])/Tk,1),reduction="batchmean")
opt.zero_grad(); loss.backward(); opt.step()
return ad.eval()
def evaluate(name,ad,ms_cal,ms_test,okTc,okTt):
okc,cc=eval_ad(ad,ms_cal); okt,ctt=eval_ad(ad,ms_test); K=ad.K
def hyb(okA,conf,okT,ms,thr):
s={m for m in ms if covered(m) and conf[m]>=thr}
return sum((okA[m] if m in s else okT[m]) for m in ms)/len(ms), sum((K if m in s else pos(m)) for m in ms)/len(ms), len(s)/len(ms)
bc=sum(okTc.values())/len(ms_cal); thr=1.01
for t in [i/100 for i in range(0,100,2)]:
if bc-hyb(okc,cc,okTc,ms_cal,t)[0]<=0.01: thr=t; break
ha,hp,hf=hyb(okt,ctt,okTt,ms_test,thr)
U=[m for m in ms_test if not covered(m)]; C=[m for m in ms_test if covered(m)]
tb=sum(okTt.values())/len(ms_test)
r=dict(iter=name,K=K,text_acc=tb,text_pos=sum(map(pos,ms_test))/len(ms_test),
native_acc_cov=sum(okt[m] for m in C)/len(C),native_acc_sysholdout=sum(okt[m] for m in U)/len(U),
text_acc_sysholdout=sum(okTt[m] for m in U)/len(U),native_all_acc=sum(okt.values())/len(ms_test),
gate_thr=thr,hyb_acc=ha,hyb_pos=hp,gate_native_frac=hf,hyb_deg_pp=100*(tb-ha),
native_cal_sysholdout=sum(okc[m] for m in ms_cal if not covered(m))/max(1,len([m for m in ms_cal if not covered(m)])))
r["promising_native"]= (100*(r["text_acc_sysholdout"]-r["native_acc_sysholdout"])<=5) and K<=0.7*r["text_pos"]
r["promising_hybrid_pos"]= hp<=0.7*r["text_pos"] and r["hyb_deg_pp"]<=2 and hf>=0.3
return r
CFG={"013c":("fact4",200,True),"013d":("add1",200,True),"013e":("mlp1",200,True),"013f":("mlp4",200,True),"013g":("fact4",600,True)}
order=sys.argv[1].split(",")
for name in order:
mode,steps,dis=CFG[name]; torch.manual_seed(0); random.seed(0)
ad=Fact(mode) if mode in("fact4","add1") else MLP(int(mode[-1]))
t0=time.time(); ad=train(ad,steps,dis)
r=evaluate(name,ad,cal_all,dev,okT_cal,okT_dev); r.update(mode=mode,steps=steps,train_s=round(time.time()-t0),params=sum(p.numel() for p in ad.parameters()))
open("iteraciones.jsonl","a").write(json.dumps(r)+"\n"); torch.save(ad.state_dict(),f"adapter_{name}.pt"); print(name,{k:(round(v,3) if isinstance(v,float) else v) for k,v in r.items()},flush=True)
if r["promising_native"] or r["promising_hybrid_pos"]:
okTf,_=eval_text(FMT,fresh); rf=evaluate(name+"_FRESH",ad,cal_all,fresh,okT_cal,okTf)
open("iteraciones.jsonl","a").write(json.dumps(rf)+"\n"); print("FRESH",rf,flush=True); break
print("DONE",flush=True)