Download reports/nnlab/verify/recompute.py from Karar-git/hack4dev-building-age-winners: direct link, hf CLI and curl.
- Browser
- Download file 4.36 kB
-
https://huggingface.co/Karar-git/hack4dev-building-age-winners/resolve/main/reports/nnlab/verify/recompute.py
- Command line
-
hf download hf://Karar-git/hack4dev-building-age-winners/reports/nnlab/verify/recompute.py
-
curl -L -o recompute.py https://huggingface.co/Karar-git/hack4dev-building-age-winners/resolve/main/reports/nnlab/verify/recompute.py
4.36 kB
| import os, sys, json, time | |
| os.environ["NNLAB_THREADS"]="2"; os.environ["OMP_NUM_THREADS"]="2"; os.environ["MKL_NUM_THREADS"]="2"; os.environ["OPENBLAS_NUM_THREADS"]="2" | |
| import numpy as np, pandas as pd | |
| from pathlib import Path | |
| from scipy import sparse | |
| from scipy.spatial import cKDTree | |
| ROOT=Path("."); PRED=ROOT/"reports/nnlab/preds"; SC=ROOT/"reports/nnlab/scores" | |
| t0=time.time() | |
| ams=pd.read_parquet(ROOT/"data/processed/amsterdam_features.parquet") | |
| mad=pd.read_parquet(ROOT/"data/processed/madrid_features.parquet", columns=["age_class"]) | |
| ya=ams["age_class"].to_numpy().astype(int); N=len(ya) | |
| pa=ams[["px_key","py_key"]].to_numpy().astype(np.int64) | |
| ym=mad["age_class"].to_numpy().astype(int) | |
| prior=np.array([(ym==c).mean() for c in (1,2,3,4)]) | |
| print("N",N,"prior",prior.round(4)) | |
| def shots(n,rep): | |
| rng=np.random.default_rng(42+1000*n+rep) | |
| return np.concatenate([rng.choice(np.flatnonzero(ya==c), size=min(n,(ya==c).sum()), replace=False) for c in (1,2,3,4)]) | |
| def ld(m,n,r): | |
| p=PRED/m/f"n{n}_r{r}.npy" | |
| return np.load(p).astype(np.float64) | |
| def mysoftmax(L): | |
| L=L-L.max(1,keepdims=True); E=np.exp(L); return E/E.sum(1,keepdims=True) | |
| def myf1(y,yh): | |
| f=[] | |
| for c in (1,2,3,4): | |
| tp=((y==c)&(yh==c)).sum(); fp=((y!=c)&(yh==c)).sum(); fn=((y==c)&(yh!=c)).sum() | |
| f.append(0 if tp==0 else 2*tp/(2*tp+fp+fn)) | |
| return float(np.mean(f)) | |
| SH=["lgbm_upw","met_protoda_maha","met_cos_maha","tab_dann_rz_ft"] | |
| DR=[(5,0),(25,1),(200,2)] | |
| # (2) own graph | |
| from sklearn.decomposition import PCA | |
| sys.path.insert(0,str(ROOT/"release/hack4dev-building-age-winners")) | |
| import winners as W | |
| feats=W.feature_names() | |
| X=ams[feats].to_numpy(np.float32).astype(np.float64) | |
| med=np.median(X,0); q75,q25=np.percentile(X,[75,25],0); iqr=np.where((q75-q25)>1e-12,q75-q25,1.0) | |
| Z=np.clip((X-med)/iqr,-10,10).astype(np.float32) | |
| Zp=PCA(32,random_state=0).fit_transform(Z) | |
| U=Zp/np.clip(np.linalg.norm(Zp,axis=1,keepdims=True),1e-12,None) | |
| _,nb=cKDTree(U).query(U,k=11) | |
| rows=np.repeat(np.arange(N),10); cols=nb[:,1:].ravel() | |
| K=sparse.coo_matrix((np.ones(len(rows)),(rows,cols)),shape=(N,N)).tocsr(); K=K.maximum(K.T) | |
| pr=cKDTree(pa.astype(float)).query_pairs(3,output_type="ndarray") | |
| Sp=sparse.coo_matrix((np.ones(len(pr)),(pr[:,0],pr[:,1])),shape=(N,N)).tocsr(); Sp=Sp+Sp.T | |
| A=Sp.maximum(K).tocsr(); A.setdiag(0); A.eliminate_zeros(); A.data[:]=1.0 | |
| Ac=sparse.load_npz(ROOT/"reports/nnlab/cache/transd_graph_ams_r3_k10_pca32.npz").tocsr() | |
| dA=(A!=Ac).nnz | |
| print(f"own graph nnz={A.nnz} cached nnz={Ac.nnz} differing entries={dA} meandeg={A.sum()/N:.2f} [{time.time()-t0:.0f}s]") | |
| deg=np.asarray(A.sum(1)).ravel(); dinv=1/np.sqrt(deg); S=sparse.diags(dinv)@A@sparse.diags(dinv) | |
| sys.path.insert(0,str(ROOT/"tools/nnlab")) | |
| import lab | |
| res={} | |
| sc=json.load(open(SC/"ens_e4__lp.json")); scd={(r["n"],r["rep"]):r for r in sc["draws"]} | |
| sc4=json.load(open(SC/"ens_e4.json")); scd4={(r["n"],r["rep"]):r for r in sc4["draws"]} | |
| for n,rep in DR: | |
| zs=np.log(np.clip(ld("zs_xgb_dann",0,0),1e-6,1)) | |
| sh=np.mean([np.log(np.clip(ld(m,n,rep),1e-6,1)) for m in SH],0) | |
| w0=10/(n+10) | |
| E=mysoftmax(w0*zs+(1-w0)*sh) | |
| Es=ld("ens_e4",n,rep) | |
| s=shots(n,rep); s_lab=lab.W.shots_for(ya,n,rep) | |
| ys=ya[s] | |
| Y0=E.copy(); Y0[s]=0; Y0[s,ys-1]=10.0 | |
| F=Y0.copy() | |
| for _ in range(20): F=0.8*(S@F)+0.2*Y0 | |
| F=np.clip(F,1e-12,None); F/=F.sum(1,keepdims=True) | |
| Fs=ld("ens_e4__lp",n,rep) | |
| keep=np.setdiff1d(np.arange(N),s) | |
| out={} | |
| for nm,P in (("ens_e4",E),("ens_e4__lp",F)): | |
| Pk=P[keep]; y=ya[keep] | |
| out[nm]=dict(argmax=myf1(y,Pk.argmax(1)+1), rule=myf1(y,lab.W.decide(Pk,prior)), | |
| rule_smooth=myf1(y,lab.W.decide(lab.smooth(Pk,pa,keep=keep),prior))) | |
| # also F1 on stored files | |
| print(f"n={n} rep={rep}: shots equal lab={np.array_equal(np.sort(s),np.sort(s_lab))} |shots|={len(s)} overlap-check uniq={len(np.unique(s))}") | |
| print(f" ens_e4 maxabs(own-stored)={np.abs(E-Es).max():.2e} ens_e4__lp maxabs(own-stored)={np.abs(F-Fs).max():.2e} argmax agree={(F.argmax(1)==Fs.argmax(1)).mean():.6f}") | |
| for nm,d_ in (("ens_e4",scd4),("ens_e4__lp",scd)): | |
| st=d_[(n,rep)] | |
| print(" "+nm.ljust(11)+" ".join(f"{k}: own={out[nm][k]:.4f} stored={st[k]:.4f}" for k in ("argmax","rule","rule_smooth"))) | |
| # leakage probe: shot-row mass of F away from shot label vs non-shots | |
| print(f"done [{time.time()-t0:.0f}s]") | |