import os, sys, json, time os.environ["NNLAB_THREADS"]="2"; os.environ["OMP_NUM_THREADS"]="2"; os.environ["MKL_NUM_THREADS"]="2"; os.environ["OPENBLAS_NUM_THREADS"]="2" import numpy as np, pandas as pd from pathlib import Path from scipy import sparse from scipy.spatial import cKDTree ROOT=Path("."); PRED=ROOT/"reports/nnlab/preds"; SC=ROOT/"reports/nnlab/scores" t0=time.time() ams=pd.read_parquet(ROOT/"data/processed/amsterdam_features.parquet") mad=pd.read_parquet(ROOT/"data/processed/madrid_features.parquet", columns=["age_class"]) ya=ams["age_class"].to_numpy().astype(int); N=len(ya) pa=ams[["px_key","py_key"]].to_numpy().astype(np.int64) ym=mad["age_class"].to_numpy().astype(int) prior=np.array([(ym==c).mean() for c in (1,2,3,4)]) print("N",N,"prior",prior.round(4)) def shots(n,rep): rng=np.random.default_rng(42+1000*n+rep) return np.concatenate([rng.choice(np.flatnonzero(ya==c), size=min(n,(ya==c).sum()), replace=False) for c in (1,2,3,4)]) def ld(m,n,r): p=PRED/m/f"n{n}_r{r}.npy" return np.load(p).astype(np.float64) def mysoftmax(L): L=L-L.max(1,keepdims=True); E=np.exp(L); return E/E.sum(1,keepdims=True) def myf1(y,yh): f=[] for c in (1,2,3,4): tp=((y==c)&(yh==c)).sum(); fp=((y!=c)&(yh==c)).sum(); fn=((y==c)&(yh!=c)).sum() f.append(0 if tp==0 else 2*tp/(2*tp+fp+fn)) return float(np.mean(f)) SH=["lgbm_upw","met_protoda_maha","met_cos_maha","tab_dann_rz_ft"] DR=[(5,0),(25,1),(200,2)] # (2) own graph from sklearn.decomposition import PCA sys.path.insert(0,str(ROOT/"release/hack4dev-building-age-winners")) import winners as W feats=W.feature_names() X=ams[feats].to_numpy(np.float32).astype(np.float64) med=np.median(X,0); q75,q25=np.percentile(X,[75,25],0); iqr=np.where((q75-q25)>1e-12,q75-q25,1.0) Z=np.clip((X-med)/iqr,-10,10).astype(np.float32) Zp=PCA(32,random_state=0).fit_transform(Z) U=Zp/np.clip(np.linalg.norm(Zp,axis=1,keepdims=True),1e-12,None) _,nb=cKDTree(U).query(U,k=11) rows=np.repeat(np.arange(N),10); cols=nb[:,1:].ravel() K=sparse.coo_matrix((np.ones(len(rows)),(rows,cols)),shape=(N,N)).tocsr(); K=K.maximum(K.T) pr=cKDTree(pa.astype(float)).query_pairs(3,output_type="ndarray") Sp=sparse.coo_matrix((np.ones(len(pr)),(pr[:,0],pr[:,1])),shape=(N,N)).tocsr(); Sp=Sp+Sp.T A=Sp.maximum(K).tocsr(); A.setdiag(0); A.eliminate_zeros(); A.data[:]=1.0 Ac=sparse.load_npz(ROOT/"reports/nnlab/cache/transd_graph_ams_r3_k10_pca32.npz").tocsr() dA=(A!=Ac).nnz print(f"own graph nnz={A.nnz} cached nnz={Ac.nnz} differing entries={dA} meandeg={A.sum()/N:.2f} [{time.time()-t0:.0f}s]") deg=np.asarray(A.sum(1)).ravel(); dinv=1/np.sqrt(deg); S=sparse.diags(dinv)@A@sparse.diags(dinv) sys.path.insert(0,str(ROOT/"tools/nnlab")) import lab res={} sc=json.load(open(SC/"ens_e4__lp.json")); scd={(r["n"],r["rep"]):r for r in sc["draws"]} sc4=json.load(open(SC/"ens_e4.json")); scd4={(r["n"],r["rep"]):r for r in sc4["draws"]} for n,rep in DR: zs=np.log(np.clip(ld("zs_xgb_dann",0,0),1e-6,1)) sh=np.mean([np.log(np.clip(ld(m,n,rep),1e-6,1)) for m in SH],0) w0=10/(n+10) E=mysoftmax(w0*zs+(1-w0)*sh) Es=ld("ens_e4",n,rep) s=shots(n,rep); s_lab=lab.W.shots_for(ya,n,rep) ys=ya[s] Y0=E.copy(); Y0[s]=0; Y0[s,ys-1]=10.0 F=Y0.copy() for _ in range(20): F=0.8*(S@F)+0.2*Y0 F=np.clip(F,1e-12,None); F/=F.sum(1,keepdims=True) Fs=ld("ens_e4__lp",n,rep) keep=np.setdiff1d(np.arange(N),s) out={} for nm,P in (("ens_e4",E),("ens_e4__lp",F)): Pk=P[keep]; y=ya[keep] out[nm]=dict(argmax=myf1(y,Pk.argmax(1)+1), rule=myf1(y,lab.W.decide(Pk,prior)), rule_smooth=myf1(y,lab.W.decide(lab.smooth(Pk,pa,keep=keep),prior))) # also F1 on stored files print(f"n={n} rep={rep}: shots equal lab={np.array_equal(np.sort(s),np.sort(s_lab))} |shots|={len(s)} overlap-check uniq={len(np.unique(s))}") print(f" ens_e4 maxabs(own-stored)={np.abs(E-Es).max():.2e} ens_e4__lp maxabs(own-stored)={np.abs(F-Fs).max():.2e} argmax agree={(F.argmax(1)==Fs.argmax(1)).mean():.6f}") for nm,d_ in (("ens_e4",scd4),("ens_e4__lp",scd)): st=d_[(n,rep)] print(" "+nm.ljust(11)+" ".join(f"{k}: own={out[nm][k]:.4f} stored={st[k]:.4f}" for k in ("argmax","rule","rule_smooth"))) # leakage probe: shot-row mass of F away from shot label vs non-shots print(f"done [{time.time()-t0:.0f}s]")