diff --git a/dw_lb/d0.py b/dw_lb/d0.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ccf89269a6c26f787cb53030e7d0b99016c0de --- /dev/null +++ b/dw_lb/d0.py @@ -0,0 +1,23 @@ +import sys; sys.path.insert(0,'/workspace/huawei/dw_lb') +from lib import * +import dwcommon as D, pickle, time +t=time.time(); d=D.Data(); print('data',time.time()-t, d.NE, d.NU, d.NI) +ec=d.Y.max(1) # event has click +print('event click rate',ec.mean()) +# user heterogeneity: split-half reliability +u=d.EU; order=np.lexsort((d.ETS,u)); +us=u[order]; bounds=np.r_[0,np.flatnonzero(np.diff(us))+1,len(us)] +n=np.diff(bounds); print('events/user mean',n.mean(),'median',np.median(n), 'min',n.min(),'max',n.max()) +ecs=ec[order]; e1=[];e2=[];ns=[] +for a,b in zip(bounds[:-1],bounds[1:]): + if b-a>=8: + h=(b-a)//2; e1.append(ecs[a:a+h].mean()); e2.append(ecs[b-h:b].mean()); ns.append(h) +e1=np.array(e1);e2=np.array(e2); print('users>=8 ev',len(e1),'corr halves',np.corrcoef(e1,e2)[0,1],'mean ns',np.mean(ns)) +# expected corr under homogeneity is 0; also compare to simulated binomial var +print('var of half-rate',e1.var(),'binomial var',np.mean(ec.mean()*(1-ec.mean())/np.array(ns))) +# by hour of day, by time since last event +ts=d.ETS; hr=((ts%DAY)//3600) +print('click rate by hour',[round(ec[hr==h].mean(),3) for h in range(24)]) +gap=np.full(len(ts),np.nan); o=order; g=np.diff(ts[o]); same=(us[1:]==us[:-1]); tmp=np.full(len(o),np.nan); tmp[1:]=np.where(same,g,np.nan); gap[o]=tmp +for lo,hi in [(0,1800),(1800,3600*3),(3*3600,6*3600),(6*3600,12*3600),(12*3600,24*3600),(24*3600,48*3600),(48*3600,1e9)]: + m=(gap>=lo)&(gap=2: g=np.diff(tss[a:bb]); gapm[uu]=g.mean(); gapsd[uu]=g.std() + ue=u[e]; tev=ts[e] + shrink=(n_cl[ue]+0.63*3)/(n_ev[ue]+3) + days=(C-J20)/DAY + col={} + col['p_nev']=n_ev[ue]; col['p_evrate']=n_ev[ue]/days; col['p_ctr']=np.where(n_ev[ue]>0,n_cl[ue]/np.maximum(n_ev[ue],1),np.nan); col['p_ctr_s']=shrink + col['p_l3']=l3[ue]; col['p_streak']=streak[ue]; col['p_gapm']=gapm[ue]/3600; col['p_gapsd']=gapsd[ue]/3600 + col['p_since_ev']=np.where(np.isfinite(last[ue]),(tev-last[ue])/3600,np.nan); col['p_since_first']=np.where(np.isfinite(first[ue]),(tev-first[ue])/3600,np.nan) + col['p_gap_ratio']=col['p_since_ev']/(col['p_gapm']+1) + # own history + hl=d.Hlen[e]; col['h_len']=hl.astype(np.float32) + hmin=np.full(nE,np.nan); hmax=np.full(nE,np.nan); hgm=np.full(nE,np.nan); hgsd=np.full(nE,np.nan); hglast=np.full(nE,np.nan); h1=np.zeros(nE); h3=np.zeros(nE); h7=np.zeros(nE); hwarm=np.zeros(nE); hwin=np.zeros(nE); huniq=np.zeros(nE) + for i,ee in enumerate(e): + a,bb=d.OFF[ee],d.OFF[ee+1] + if bb-a==0: continue + t=d.HT[a:bb]; it=d.HI[a:bb]; tt=tev[i] + hmin[i]=(tt-t[0])/3600; hmax[i]=(tt-t[-1])/3600; huniq[i]=len(set(it.tolist())) + h1[i]=(t>tt-DAY).sum(); h3[i]=(t>tt-3*DAY).sum(); h7[i]=(t>tt-7*DAY).sum(); hwarm[i]=(t=J20).sum() + if bb-a>=2: g=np.diff(t); hgm[i]=g.mean()/3600; hgsd[i]=g.std()/3600; hglast[i]=g[-1]/3600 + col.update(h_span=hmin,h_since_last=hmax,h_gap_mean=hgm,h_gap_sd=hgsd,h_gap_last=hglast,h_1d=h1,h_3d=h3,h_7d=h7,h_warm=hwarm,h_win=hwin,h_uniq=huniq,h_uniq_ratio=huniq/np.maximum(hl,1)) + col['h_rate_day']=np.where(hl>=2,(hl-1)/np.maximum(hmin-hmax,1e-3)*24,np.nan) + # clicks per event: window clicks (in train, ts0,n_ev[ue]/np.maximum((C-J20)/3600/24,1e-3),np.nan) # events per day + col['implied_cpe']=col['h_rate_day']/(erate+0.5) + col['gap_over_hgap']=col['p_gapm']/(col['h_gap_mean']+1) + X=np.column_stack([np.repeat(np.asarray(col[k],np.float32),10) for k in col]).astype(np.float32) + return X,list(col) +if __name__=='__main__': + d=D.Data() + for c in [int(x) for x in sys.argv[1].split(',')]: + F=load_cut(c); v=d.val(c); assert (v['rowid']==F['rowid']).all() + X,names=build(d,c,F); np.save(f'/workspace/huawei/dw_lb/cache_prop_c{c}.npy',X); json.dump(names,open('/workspace/huawei/dw_lb/prop_names.json','w')) + print(c,X.shape,flush=True) diff --git a/dw_lb/feat_tx.py b/dw_lb/feat_tx.py new file mode 100644 index 0000000000000000000000000000000000000000..872bc227100a94da62da3d92eb85677cd2cf5bf9 --- /dev/null +++ b/dw_lb/feat_tx.py @@ -0,0 +1,115 @@ +"""Priority 5 (TRANSDUCTIVE, label-free): statistics over the unlabelled TARGET rows (validation first-event rows when validating; evaluation rows for v14). +Uses only: target slates (10 candidate items), target event ts, target histories (items+ts), and train-side exposure/CTR strictly before C (labels of ts slates, ts, histories (as unlabeled inputs)""" + S=d.RI[e]; ts=d.ETS[e]; hi=[d.HI[d.OFF[x]:d.OFF[x+1]] for x in e]; ht=[d.HT[d.OFF[x]:d.OFF[x+1]] for x in e] + return S,ts,hi,ht +def zrk(x): + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + return ((m-mu)/sd).ravel(),(m-mu).ravel(),m.argsort(1).argsort(1).astype(np.float32).ravel() +def build_tx(S,ts,hi,ht,C,ref,NI,fams=FAMS): + n=len(S); col={}; fl=S.ravel(); rep=lambda a:np.repeat(np.asarray(a,np.float32),10) + cnt=np.bincount(fl,minlength=NI).astype(np.float32) # exposure in target slates (includes own slot) + # ---- family 1: item exposure in target slates + if 'f1' in fams: + c1=cnt[fl]-1; col['f1_cnt']=c1/n*1e4 + o=np.argsort(ts,kind='stable'); tss=ts[o] + # time-windowed exposure: key=(item,ts) of every target slot + its=np.repeat(S[o].ravel(),1); kt=np.repeat(tss,10); key=np.sort(its.astype(np.int64)*(1<<33)+kt) + base=fl.astype(np.int64)*(1<<33); tr=np.repeat(ts,10).astype(np.int64) + for nm,w in (('h1',3600),('h6',6*3600),('h24',24*3600)): + lo=np.searchsorted(key,base+tr-w,'left'); hi_=np.searchsorted(key,base+tr+w,'right'); cw=(hi_-lo-1).astype(np.float32) + nev=(np.searchsorted(tss,ts+w,'right')-np.searchsorted(tss,ts-w,'left')).astype(np.float32) # #target events in the window + col['f1_cnt_'+nm]=cw/np.repeat(nev,10)*1e3 # per-event normalised local exposure share + if nm=='h6': col['f1_cnt_h6_raw']=cw + expl=ref['ex_l'][fl]/max(ref['nL'],1)*1e3; col['f1_late_exp']=expl; col['f1_late_ctr']=(ref['cl_l'][fl]+0.2)/(ref['ex_l'][fl]+2) + col['f1_ratio_late']=(c1/n+1e-4)/((ref['ex_l'][fl]+0.5)/max(ref['nL'],1)); col['f1_ratio_all']=(c1/n+1e-4)/((ref['ex_all'][fl]+0.5)/max(ref['nX'],1)) + col['f1_exp_x_ctr']=col['f1_cnt']*col['f1_late_ctr'] + for k in ('f1_cnt','f1_cnt_h6','f1_ratio_late'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + col['f1_slate_mean']=np.repeat(col['f1_cnt'].reshape(-1,10).mean(1),10) + # ---- family 3: popularity in the histories of target users (items clicked; incl. post-cutoff clicks visible in histories) + hlen=np.array([len(x) for x in hi]); hI=np.concatenate(hi).astype(np.int64) if hlen.sum() else np.zeros(0,np.int64); hT=np.concatenate(ht).astype(np.int64) + hu=np.repeat(np.arange(n),hlen); pos_from_end=np.concatenate([np.arange(l)[::-1] for l in hlen]) if hlen.sum() else np.zeros(0) + if 'f3' in fams: + hc=np.bincount(hI,minlength=NI).astype(np.float32); col['f3_hpop']=hc[fl]/n*1e4 + wl=np.bincount(hI,weights=(pos_from_end<3)/(1.0+pos_from_end),minlength=NI).astype(np.float32); col['f3_hpop_l3']=wl[fl]/n*1e4 + l1=np.bincount(hI[pos_from_end==0],minlength=NI).astype(np.float32); col['f3_hpop_l1']=l1[fl]/n*1e4 + post=hT>=C; hp=np.bincount(hI[post],minlength=NI).astype(np.float32); col['f3_hpost']=hp[fl]/n*1e4 + hk=np.sort(hI*(1<<33)+hT); base=fl.astype(np.int64)*(1<<33); tr=np.repeat(ts,10).astype(np.int64) + for nm,w in (('6h',6*3600),('24h',24*3600)): + lo=np.searchsorted(hk,base+tr-w,'left'); hi_=np.searchsorted(hk,base+tr,'left'); col['f3_hrecent_'+nm]=(hi_-lo).astype(np.float32) # others' clicks on j in the w hours before this event (visible in target histories) + lo2=np.searchsorted(hk,base+tr,'left'); hi2=np.searchsorted(hk,base+tr+w,'right'); col['f3_hnext_'+nm]=(hi2-lo2).astype(np.float32) + # own-history subtraction for the "post" count: own clicks of j in own history + own=np.zeros(n*10,np.float32) + for i in range(n): + if hlen[i]: + h=hi[i]; own[i*10:(i+1)*10]=(S[i][:,None]==h[None,:]).sum(1) + col['f3_own_cnt']=own; col['f3_hpop_excl_own']=col['f3_hpop']-own/n*1e4 + for k in ('f3_hpop','f3_hpost','f3_hrecent_24h'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + col['f3_hpost_vs_train_ctr']=col['f3_hpost']/(ref['cl_all'][fl]/max(ref['nX'],1)*1e4+1.0) + # ---- family 2: co-occurrence in target slates + last-click share among target users + if 'f2' in fams: + B=csr_matrix((np.ones(n*10,np.float32),(np.repeat(np.arange(n),10),fl)),shape=(n,NI)); CC=(B.T@B).tocsr() + ri=np.repeat(S,10,axis=1).ravel(); ci=np.tile(S,(1,10)).ravel() # all 100 pairs per event + v=np.asarray(CC[ri,ci]).ravel().reshape(n,10,10)-1.0 # LOO own + offd=~np.eye(10,dtype=bool); cj=(cnt[S]-1) # (n,10) + norm=np.sqrt((cj[:,:,None]+1)*(cj[:,None,:]+1)) + coo=(v/norm)[:,offd].reshape(n,10,9) if False else None + vm=np.where(offd[None],v/norm,0).sum(2)/9; col['f2_cooc_norm']=vm.ravel() + col['f2_cooc_raw']=(np.where(offd[None],v,0).sum(2)/9).ravel() + # last-click share: users whose last click was h -> share of slates containing j + last=np.array([x[-1] if len(x) else -1 for x in hi],np.int64); l3=[x[-3:] for x in hi] + H=csr_matrix((np.ones((last>=0).sum(),np.float32),(np.flatnonzero(last>=0),last[last>=0])),shape=(n,NI)); A=(H.T@B).tocsr(); nh=np.asarray(H.sum(0)).ravel() + lsh=np.full((n,10),np.nan,np.float32) + ok=np.flatnonzero(last>=0) + if len(ok): + a=np.asarray(A[np.repeat(last[ok],10),S[ok].ravel()]).ravel().reshape(-1,10)-1.0; den=np.maximum(nh[last[ok]]-1,0)[:,None] + lsh[ok]=np.where(den>0,a/np.maximum(den,1),np.nan) + col['f2_lastshare_n']=np.repeat(np.where(nh[last]>0,nh[last]-1,np.nan),10).astype(np.float32) + col['f2_lastshare']=lsh.ravel() + # any of last-3 clicks (weighted share) + H3=csr_matrix((np.concatenate([np.ones(len(x),np.float32) for x in l3]),(np.repeat(np.arange(n),[len(x) for x in l3]),np.concatenate(l3).astype(np.int64) if sum(len(x) for x in l3) else [])),shape=(n,NI)); A3=(H3.T@B).tocsr(); nh3=np.asarray(H3.sum(0)).ravel() + s3=np.zeros((n,10),np.float32); d3=np.zeros(n,np.float32) + for k in range(3): + idx=[i for i in range(n) if len(l3[i])>k] + if not idx: continue + idx=np.array(idx); h=np.array([l3[i][-1-k] for i in idx],np.int64) + a=np.asarray(A3[np.repeat(h,10),S[idx].ravel()]).ravel().reshape(-1,10)-1.0 + s3[idx]+=a; d3[idx]+=np.maximum(nh3[h]-1,0) + col['f2_l3share']=np.where(d3[:,None]>0,s3/np.maximum(d3[:,None],1),np.nan).ravel() + for k in ('f2_cooc_norm','f2_lastshare','f2_l3share'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + # ---- family 4: timing among target events + if 'f4' in fams: + o=np.argsort(ts,kind='stable'); rk_=np.empty(n); rk_[o]=np.arange(n); tss=ts[o] + col['f4_trel_h']=rep((ts-C)/3600); col['f4_trank']=rep(rk_/n) + col['f4_n_same_h']=rep(np.searchsorted(tss,ts+1800,'right')-np.searchsorted(tss,ts-1800,'left')) + col['f4_n_6h']=rep(np.searchsorted(tss,ts+6*3600,'right')-np.searchsorted(tss,ts-6*3600,'left')) + col['f4_n_before_frac']=rep(rk_/n) + col['f4_hist_n']=rep(hlen) + names=list(col); X=np.column_stack([np.asarray(col[k],np.float32) for k in names]).astype(np.float32) + return X,names +def build_val(d,c,fams=FAMS): + C=Jd(c); v=d.val(c); e=v['e'][::10]; S,ts,hi,ht=target_from_data(d,e) + # sanity: slate order must match val row order + S=v['item'].reshape(-1,10).astype(np.int64) # slate in val-row order + assert (np.sort(S,1)==np.sort(d.RI[e],1)).all() + return build_tx(S,ts,hi,ht,C,train_ref(d,C),d.NI,fams) +if __name__=='__main__': + d=D.Data() + for c in [int(x) for x in sys.argv[1].split(',')]: + X,names=build_val(d,c); np.save(f'/workspace/huawei/dw_lb/cache_tx_c{c}.npy',X); json.dump(names,open(f'/workspace/huawei/dw_lb/tx_names.json','w')); print(c,X.shape,flush=True) diff --git a/dw_lb/findings.md b/dw_lb/findings.md new file mode 100644 index 0000000000000000000000000000000000000000..d88e4f791186ce71cf06091431e0fa7e78334380 --- /dev/null +++ b/dw_lb/findings.md @@ -0,0 +1,35 @@ +# dw_lb findings — what could close 0.818 → 0.83? (Grok Bot / Eric, 2026-10-28) +All Δ = global AUC vs a control model (same residual LGB on v12 logits with z-derived context only), user-grouped 5-fold CV, event bootstrap 95% CI. v12 val logits reproduced exactly (C25 0.81723, C23 0.79566). Null-noise feature: Δ −0.00006 (C25), 0.00000 (C23) → harness false-positive floor ≈ ±0.0001. + +## 1. Public intel (honest: nothing concrete exists) +- Official description only: "Competing in the Intelligent Marketplace: Generative Sequential Recommendation Under Constraints" — feedback loop, multiple agents competing, compute/runtime constraints. https://huaweiireland.teamtailor.com/jobs/8447633-huawei-ireland-tech-arena-2026 , https://www.siliconrepublic.com/machines/how-huaweis-tech-arena-2026-aims-to-develop-irelands-ai-talent , https://www.hackathon.com/event/huawei-ireland-tech-arena-2026-6ac54367612836bf413df387 (Phase 1 Oct 21–Nov 4, top 15 announced Nov 9). +- No public Phase-1 solutions, leaderboard write-ups or CTR/AUC specifics found. The 0.83 figure is hearsay; unverifiable. Older edition posts (LinkedIn, e.g. https://www.linkedin.com/posts/rahulraj1406_huawei-techarena-llm-activity-7397952145213292544-JDot) concern a different task. + +## 2. Ranked ΔAUC (C25 / C23) +| # | lever | C25 | C23 | verdict | +|---|---|---|---|---| +| 1 | User click-propensity (28 feats, feat_prop.py) | +0.00020 [+0.00008,+0.00033] | +0.00026 [+0.00012,+0.00043] | real but tiny | +| 2 | Transductive family f2 co-occurrence (VAL only) | +0.00013 [+0.00004,+0.00023] (seed1 +0.00012) | −0.00005 | ≈0 | +| 3 | Transductive f3 history popularity | +0.00010 / +0.00013 | +0.00009 | ≈0 (CI spans 0) | +| 4 | Transductive all 60 | +0.00010 [−0.00008,+0.00024] | +0.00012 [−0.00004,+0.00028] | ≈0 | +| 5 | Transductive f1 exposure | −0.00002 / −0.00005 | +0.00008 | 0 | +| 6 | Transductive f4 timing | +0.00003 / +0.00005 | +0.00007 | 0 | +| 7 | transductive + propensity (88) | +0.00018 [−0.00001,+0.00033] | +0.00029 [+0.00013,+0.00047] | = propensity alone | +| 8 | Two-stage event model P(ev)·softmax (ctx / ctx+prop) | glob 0.81707 / 0.81720 vs z 0.81723 | 0.79508 / 0.79525 vs 0.79566 | negative | +| 9 | More training events (learning curve, 1→4 pseudo-folds, same LGB, cut 25) | 0.8142 / 0.8158 / 0.8162 / 0.8160 | – | saturated after ~2–3 folds → all-events rebuild est. ≤ +0.001 | +| 10 | From-scratch softmax+outside-option latent model (torch, all events) | alone 0.58–0.60 | – | useless: sparse (50k items, 53k users, 687k events), overfits | + +## 3. Simulator facts learned +- **Each event has at most ONE click** (0.366 no-click, 0.634 one click, nothing else) → single-choice softmax with an outside option; v12's binary logits sum to 0.98 per slate vs 0.63 observed (calibrated within-event ranking is what matters). +- P(no click) falls with slate logsumexp but is more consistent with a taste-dependent outside option than a constant one: constant-outside prediction overshoots P0 (e.g. top decile pred 0.31 vs obs 0.17). +- Event "any click" AUC: lse 0.668 → +z-context LGB 0.671 → + user propensity 0.672 (C25); C23 0.644→0.649→0.652. Oracle event label would give 0.894 (C25). So the 0.83 gap would have to come from event-level prediction ≈0.74+ (headroom sim: event AUC 0.76 ⇒ +0.0075), which is not reachable from user propensity (split-half user corr only 0.39, event click rate flat in hour/gap) nor from slate/exposure aggregates. +- Transductive univariates ≤0.53 within-event AUC; stationary data; exposure unrelated to click; post-cutoff clicks in histories ≈0.500. + +## 4. Which lever most likely explains 0.83? +None of the five measured levers can: sum of all positive finds ≈ +0.0005–0.001. The gap (if real) is most plausibly better **item–user taste modelling** (a better preference model improves both within-event ranking and event-level lse), not data volume, propensity, or eval aggregates. My own end-to-end latent softmax failed because of sparsity, so any win there needs the strong CF-style features v12 already has (+ ensembling of many diverse models). Cheap remaining idea within v12's paradigm: more diverse seeds/learners (blend gains in v12 notes were ~+0.001 each). + +## 5. Value of transductive features (information only, for Eric) +Measured cost of the eval-aggregate ban: **≈ +0.0001 (CI spans 0) on C25/C23** — not worth lifting the ban. v14 README written in /workspace/huawei/v14_transductive/, submission NOT built (nothing gained). + +## 6. Code paths (all in /workspace/huawei/dw_lb/) +lib.py (z12, load_cut, oof, boot_delta, report) · feat_prop.py → cache_prop_c{25,23}.npy (propensity, ready to hand to v12 as new feature family; leak-safe: snapshot ts=1: return df + return df.filter((pl.col('u').hash(seed=7) % 10000) < int(frac*10000)) +FL=[None] +def load_tab_rows(n, rid, extra=(), variant='notime'): + """same as load_tab but restricted to rowids rid (polars Series) -> low memory""" + a=pl.scan_parquet(f'{P3}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + b=pl.scan_parquet(f'{P2}/{V2_OF[n]}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.scan_parquet(f'{D}/{sub}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect(),on='rowid',how='left') + return a +def build_chunked(n, extra, variant, frac, nch=4): + rid_all=pl.read_parquet(f'{P3}/{n}.parquet',columns=['rowid'])['rowid']; parts=[] + step=(len(rid_all)+nch-1)//nch + for i in range(nch): + rid=rid_all.slice(i*step,step); t=usample(load_tab_rows(n,rid,extra,variant),frac) + # restore original order within chunk + t=pl.DataFrame({'rowid':rid}).join(t,on='rowid',how='inner',maintain_order='left') + fl=featlist(t,variant); FL[0]=fl + t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl))) + t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP]) + parts.append(t); gc.collect() + return pl.concat(parts) +BASE=('feat_lr2','feat_tt','feat_pp') +def matrix(n, frac=1.0, variant='notime', extra=()): + """v9: base = cached v8-lite matrix (read-only, from v9d/val/cache_mx); new dirs (feat_tm*) joined on rowid. + returns (X float32, meta DataFrame, feats).""" + base=tuple(e for e in extra if e in BASE); new=[e for e in extra if e not in BASE]; assert base==BASE, extra + key=hashlib.md5(json.dumps([n,frac,variant,list(base),os.path.getmtime(f'{P3}/{n}.parquet')]).encode()).hexdigest()[:12] + fp=f'{CACHE}/{n}_{key}.parquet'; assert os.path.exists(fp), fp + fl=json.load(open(fp+'.json')); t=pl.read_parquet(fp) + for sub in new: + e=pl.read_parquet(f'{D}/{sub}/{n}.parquet'); nc=[c for c in e.columns if c!='rowid' and c not in fl] + t=t.join(e.select(['rowid']+nc),on='rowid',how='left',maintain_order='left'); fl=fl+nc + drop=os.environ.get('V9DROP') + if drop: + import re; fl=[c for c in fl if not re.search(drop,c)] + X=t.select(fl).cast(pl.Float32).to_numpy(); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect() + return X, meta, fl diff --git a/dw_lb/lc/ens.py b/dw_lb/lc/ens.py new file mode 100644 index 0000000000000000000000000000000000000000..a9161a4343f92cba8daa4a4a244b84fd69e07522 --- /dev/null +++ b/dw_lb/lc/ens.py @@ -0,0 +1,41 @@ +"""Phase 1 ensemble components. python ens.py CUT TAG [--seeds 0,1] [--folds f|fe] [--stale] [--rc] [--drop re] [--params json] [--rounds 400] +CUT=25/26: train f1..f(CUT-21) (+e/d folds) full data, predict val{CUT}; CUT=27: train f1..f6 (+E1..E6), predict eval. +Saves ens/{tag}_c{CUT}_s{seed}.npy with raw scores at rounds 300,400(,500) (columns).""" +import sys, re, json, gc, time, argparse, numpy as np, lightgbm as lgb +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--seeds',default='0') +ap.add_argument('--folds',default='f'); ap.add_argument('--stale',action='store_true'); ap.add_argument('--rc',action='store_true') +ap.add_argument('--drop',default=None); ap.add_argument('--params',default='{}'); ap.add_argument('--rounds',type=int,default=400); ap.add_argument('--threads',type=int,default=5); ap.add_argument('--lr',default=''); ap.add_argument('--nfolds',type=int,default=99); ap.add_argument('--negfrac',type=float,default=1.0); ap.add_argument('--keep',default=''); ap.add_argument('--wtag',default=''); ap.add_argument('--wpow',type=float,default=1.0); ap.add_argument('--agew',type=float,default=0); ap.add_argument('--save',default='300,400,500') +a=ap.parse_args(); t0=time.time() +extra=(('feat_rc',) if a.rc else ())+tuple(x for x in a.lr.split(',') if x); var='stale' if a.stale else 'notime' +efold={23:[],25:[f'e{k}' for k in range(1,5)],26:[f'd{k}' for k in range(1,6)],27:[f'E{k}' for k in range(1,7)]}[a.cut] +trn=([f'f{k}' for k in range(max(1,a.cut-20-a.nfolds),a.cut-20)] if 'f' in a.folds else [])+(efold if 'e' in a.folds else []) +import polars as pl +lens=[pl.read_parquet_schema and len(pl.read_parquet(f'feat_v3/{n}.parquet',columns=['rowid'])) for n in trn] +rs=np.random.RandomState(123); R=rs.rand(sum(lens)) if a.negfrac<1 else None +Xs=[];ys=[];ws=[];feats=None;off=0 +for n,L in zip(trn,lens): + X,m,f=matrix(n,1.0,variant=var,extra=extra); feats=f; yy=m['clicked'].to_numpy(); assert len(yy)==L + sel=(yy==1)|(R[off:off+L]&1 | grep -v Warn | tail -2 + done +done +echo LC_DONE diff --git a/dw_lb/lib.py b/dw_lb/lib.py new file mode 100644 index 0000000000000000000000000000000000000000..fd28528ba6fb8541fd41fb5164356e7fc6d8ea5e --- /dev/null +++ b/dw_lb/lib.py @@ -0,0 +1,43 @@ +import sys, os, json, numpy as np, polars as pl, lightgbm as lgb, warnings; warnings.filterwarnings('ignore') +sys.dont_write_bytecode=True +sys.path.insert(0,'/workspace/huawei/dw_levers'); sys.path.insert(1,'/workspace/huawei/dw_signals') +from sklearn.metrics import roc_auc_score as A +from sklearn.model_selection import GroupKFold +H='/workspace/huawei'; DAY=86400; J20=1737331200 +def Jd(d): return J20+(d-20)*DAY +def z12(c): + s=lambda t,k:np.load(f'{H}/v12/ens/{t}_c{c}_s0.npy')[:,k] + m=np.mean([np.load(f'{H}/v12/ens/dwmlp_c{c}_s{k}.npy')[:,3] for k in (range(3) if c!=26 else [0])],0) + return ((s('dwbig',1)+s('dw_s',1)+2*m)/4).astype(np.float64) +def meta(c): return pl.read_parquet(f'{H}/v10/feat_v3/val{c}.parquet',columns=['rowid','eid','u','ts','clicked']) +def base_ctx(z): + Z=z.reshape(-1,10); mu=Z.mean(1,keepdims=True); sd=Z.std(1,keepdims=True)+1e-6 + rk=Z.argsort(1).argsort(1).astype(np.float32) + lse=np.log(np.exp(Z).sum(1,keepdims=True)) + cols=[z,((Z-mu)/sd).ravel(),(Z-mu).ravel(),rk.ravel(),np.repeat(lse,10,1).ravel(),np.repeat(Z.max(1,keepdims=True),10,1).ravel(),np.repeat(mu,10,1).ravel(),np.repeat(sd,10,1).ravel()] + return np.column_stack(cols).astype(np.float32) +PI=dict(objective='binary',learning_rate=0.03,num_leaves=15,min_data_in_leaf=500,feature_fraction=0.8,bagging_fraction=0.8,bagging_freq=1,lambda_l2=30,max_bin=127,verbose=-1,num_threads=4) +def oof(X,y,z,user,seed=0,rounds=200,k=5,params=None): + p=np.zeros(len(y)); P={**PI,**(params or {})} + for tr,te in GroupKFold(k).split(X,y,user): + m=lgb.train({**P,'seed':seed},lgb.Dataset(X[tr],y[tr],init_score=z[tr]),rounds); p[te]=m.predict(X[te],raw_score=True)+z[te] + return p +def boot_delta(y,p0,p1,ev,nb=200,seed=0): + """bootstrap over events (10 rows each) of AUC(p1)-AUC(p0)""" + rs=np.random.RandomState(seed); n=len(y)//10; Y=y.reshape(n,10); a=p0.reshape(n,10); b=p1.reshape(n,10); out=[] + for _ in range(nb): + i=rs.randint(0,n,n); yy=Y[i].ravel(); out.append(A(yy,b[i].ravel())-A(yy,a[i].ravel())) + return np.percentile(out,[2.5,97.5]), np.std(out) +def load_cut(c): + m=meta(c); z=z12(c); assert len(z)==m.height + return dict(c=c,y=m['clicked'].to_numpy().astype(int),z=z,user=m['u'].to_numpy(),rowid=m['rowid'].to_numpy(),ts=m['ts'].to_numpy(),Xb=base_ctx(z)) +def report(name,F,Xnew,seeds=(0,),log='results.jsonl',control_cols=None): + """F: cut dict; Xnew: (n,k) new feats. control = Xb only; test = Xb+Xnew. returns delta vs z and vs control""" + y,z,u=F['y'],F['z'],F['user']; res=dict(name=name,cut=F['c'],nfeat=Xnew.shape[1],base=A(y,z)) + pc=np.mean([oof(F['Xb'],y,z,u,s) for s in seeds],0); X=np.hstack([F['Xb'],Xnew]); pt=np.mean([oof(X,y,z,u,s) for s in seeds],0) + res.update(control=A(y,pc),test=A(y,pt)); res['d_vs_z']=res['test']-res['base']; res['d_vs_ctrl']=res['test']-res['control'] + ci,sd=boot_delta(y,pc,pt,None); res['ci_vs_ctrl']=[float(ci[0]),float(ci[1])]; res['sd']=float(sd) + ev=lambda p: A((y.reshape(-1,10).max(1)),np.log(np.exp(p.reshape(-1,10)).sum(1))) + res['evAUC_base']=ev(z); res['evAUC_test']=ev(pt) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in res.items()}),flush=True) + open(f'/workspace/huawei/dw_lb/{log}','a').write(json.dumps(res)+'\n'); return res,pt diff --git a/dw_lb/p4.log b/dw_lb/p4.log new file mode 100644 index 0000000000000000000000000000000000000000..99657fc892665659dffe1b88a2bee46c4c812201 --- /dev/null +++ b/dw_lb/p4.log @@ -0,0 +1,2 @@ +{"cut": 25, "ev_lse": 0.66805, "ev_max": 0.62214, "ev_ctx": 0.67124, "ev_ctx_prop": 0.6723, "glob_ctx": 0.81707, "glob_ctx_prop": 0.8172, "glob_z": 0.81723, "glob_oracle_ev": 0.89396} +{"cut": 23, "ev_lse": 0.64445, "ev_max": 0.60332, "ev_ctx": 0.6488, "ev_ctx_prop": 0.65235, "glob_ctx": 0.79508, "glob_ctx_prop": 0.79525, "glob_z": 0.79566, "glob_oracle_ev": 0.88174} diff --git a/dw_lb/p4_softmax.py b/dw_lb/p4_softmax.py new file mode 100644 index 0000000000000000000000000000000000000000..0265c1b980ef77f6dce5d251df0bfe35e701d009 --- /dev/null +++ b/dw_lb/p4_softmax.py @@ -0,0 +1,56 @@ +"""Priority 4: parametric single-choice softmax + user-specific outside option, trained end-to-end on ALL events with ts + b_j + g(match(j in history)); outside logit o = MLP(u,...). P(click j)=exp(s_j)/(exp(o)+sum exp s_k). Outputs val logit log P(click j).""" +import sys, time; sys.path.insert(0,'/workspace/huawei/dw_lb') +from lib import * +import torch, torch.nn as nn, torch.nn.functional as Fn +import dwcommon as D +torch.set_num_threads(4); import os; USEU=int(os.environ.get('USEU','1')); TIE=int(os.environ.get('TIE','0')) + +def pad_hist(d,ev,L=20): + n=len(ev); Hp=np.full((n,L),-1,np.int64); Tp=np.zeros((n,L),np.float32) + for i,e in enumerate(ev): + a,b=d.OFF[e],d.OFF[e+1]; k=b-a + if k: Hp[i,L-k:]=d.HI[a:b]; Tp[i,L-k:]=(d.ETS[e]-d.HT[a:b])/3600. + return Hp,Tp +class M(nn.Module): + def __init__(s,NU,NI,k=32): + super().__init__(); s.Eu=nn.Embedding(NU,k); s.Ei=nn.Embedding(NI,k); s.Eh=nn.Embedding(NI+1,k,padding_idx=NI); s.bi=nn.Embedding(NI,1) + s.pw=nn.Parameter(torch.zeros(20)); s.proj=nn.Linear(k,k); s.match=nn.Sequential(nn.Linear(4,16),nn.ReLU(),nn.Linear(16,1)) + s.out=nn.Sequential(nn.Linear(k+3,32),nn.ReLU(),nn.Linear(32,1)); s.k=k + for e in (s.Eu,s.Ei,s.Eh): nn.init.normal_(e.weight,0,0.05) + nn.init.zeros_(s.bi.weight) + def forward(s,u,S,Hp,Tp,gap): + if TIE: s.Eh.weight.data[:-1]=s.Ei.weight.data if False else s.Eh.weight.data[:-1] + NI=s.Ei.num_embeddings; valid=(Hp>=0); Hq=torch.where(valid,Hp,torch.full_like(Hp,NI)) + w=torch.softmax(s.pw[None]+torch.where(valid,0.,-1e4),1)*valid.any(1,keepdim=True) + pool=(s.Eh(Hq)*w[:,:,None]).sum(1); uv=(s.Eu(u) if USEU else 0)+s.proj(pool) + sc=(s.Ei(S)*uv[:,None,:]).sum(2)+s.bi(S).squeeze(2) + m=(S[:,:,None]==Hp[:,None,:])&valid[:,None,:]; cnt=m.sum(2).float(); pos=torch.where(m,torch.arange(20,device=S.device)[None,None,:].float(),torch.full((1,),-1.,device=S.device)).max(2).values; rec=torch.where(cnt>0,(pos+1)/20.,torch.zeros_like(pos)) + age=torch.where(m,Tp[:,None,:],torch.zeros_like(Tp[:,None,:])).sum(2)/torch.clamp(cnt,min=1) + mf=torch.stack([(cnt>0).float(),torch.log1p(cnt),rec,torch.log1p(age)/5],2); sc=sc+s.match(mf).squeeze(2) + hl=valid.sum(1,keepdim=True).float() + o=s.out(torch.cat([uv,hl/20,torch.log1p(gap)[:,None]/5,(hl==0).float()],1)).squeeze(1) + return sc,o +def run(c,epochs=6,k=32,seed=0,lam=1e-2,lr=3e-3,tag=''): + torch.manual_seed(seed); d=D.Data(); C=Jd(c); nX=int(np.searchsorted(d.ETS,C)); v=d.val(c); ev=v['e'][::10] + tr=np.arange(nX); Htr,Ttr=pad_hist(d,tr); Hv,Tv=pad_hist(d,ev) + gap=lambda e: np.zeros(len(e),np.float32) + ys=d.Y[:nX]; lab=np.where(ys.sum(1)>0,ys.argmax(1)+1,0) # 0=no click(outside) + S=torch.tensor(d.RI[:nX].astype(np.int64)); U=torch.tensor(d.EU[:nX].astype(np.int64)); Hp=torch.tensor(Htr); Tp=torch.tensor(Ttr); L=torch.tensor(lab); G=torch.zeros(nX) + Sv=torch.tensor(v['item'].reshape(-1,10).astype(np.int64)); Uv=torch.tensor(v['user'][::10].astype(np.int64)); Hvt=torch.tensor(Hv); Tvt=torch.tensor(Tv); Gv=torch.zeros(len(ev)) + m=M(d.NU,d.NI,k); opt=torch.optim.Adam(m.parameters(),lr=lr); yv=v['y'] + def predict(): + m.eval(); out=[] + with torch.no_grad(): + for i in range(0,len(ev),8192): + sc,o=m(Uv[i:i+8192],Sv[i:i+8192],Hvt[i:i+8192],Tvt[i:i+8192],Gv[i:i+8192]); lse=torch.logsumexp(torch.cat([o[:,None],sc],1),1,keepdim=True); out.append((sc-lse).numpy()) + m.train(); return np.concatenate(out) + bs=2048; t0=time.time() + for epc in range(epochs): + perm=torch.randperm(nX) + for i in range(0,nX,bs): + b=perm[i:i+bs]; sc,o=m(U[b],S[b],Hp[b],Tp[b],G[b]); logit=torch.cat([o[:,None],sc],1); loss=Fn.cross_entropy(logit,L[b])+lam*((m.Eu(U[b])**2).sum(1).mean()+(m.Ei(S[b])**2).sum(2).mean()+(m.Eh.weight[Hp[b].clamp(min=0)]**2).sum(2).mean()) + opt.zero_grad(); loss.backward(); opt.step() + p=predict(); print(f'c{c} ep{epc} loss {loss.item():.4f} val AUC logP {A(yv,p.ravel()):.5f} t={time.time()-t0:.0f}s',flush=True) + np.save(f'/workspace/huawei/dw_lb/cache_sm{tag}_c{c}.npy',p.ravel().astype(np.float32)) +if __name__=='__main__': run(int(sys.argv[1]),int(sys.argv[2]),int(sys.argv[3]),0,float(sys.argv[4]),float(sys.argv[5]),sys.argv[6] if len(sys.argv)>6 else '') diff --git a/dw_lb/prop23.log b/dw_lb/prop23.log new file mode 100644 index 0000000000000000000000000000000000000000..c616ead111aa80b26aab2851133bd17085ac1d02 --- /dev/null +++ b/dw_lb/prop23.log @@ -0,0 +1 @@ +23 (501550, 28) diff --git a/dw_lb/prop_names.json b/dw_lb/prop_names.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe1bbe520238e8a23d35c9081a83ba1cb3864bd --- /dev/null +++ b/dw_lb/prop_names.json @@ -0,0 +1 @@ +["p_nev", "p_evrate", "p_ctr", "p_ctr_s", "p_l3", "p_streak", "p_gapm", "p_gapsd", "p_since_ev", "p_since_first", "p_gap_ratio", "h_len", "h_span", "h_since_last", "h_gap_mean", "h_gap_sd", "h_gap_last", "h_1d", "h_3d", "h_7d", "h_warm", "h_win", "h_uniq", "h_uniq_ratio", "h_rate_day", "p_clk_per_ev", "implied_cpe", "gap_over_hgap"] \ No newline at end of file diff --git a/dw_lb/results.jsonl b/dw_lb/results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6b503ff6017718472f7dd3d2e64386f652ed59aa --- /dev/null +++ b/dw_lb/results.jsonl @@ -0,0 +1,4 @@ +{"name": "null_event_noise", "cut": 25, "nfeat": 1, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8171577530714159, "d_vs_z": -6.819451209449934e-05, "d_vs_ctrl": -6.17451939234348e-05, "ci_vs_ctrl": [-0.00010993807087290585, -7.890079290281935e-06], "sd": 2.8203555989303043e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6656587077481898} +{"name": "prop_all", "cut": 25, "nfeat": 28, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8174189331936028, "d_vs_z": 0.0001929856100923999, "d_vs_ctrl": 0.00019943492826346443, "ci_vs_ctrl": [8.425431766705683e-05, 0.00032945002107639954], "sd": 6.277318634322474e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6671521893599606} +{"name": "null_event_noise", "cut": 23, "nfeat": 1, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7955480279490578, "d_vs_z": -0.00010774943840785589, "d_vs_ctrl": 2.5894856480190853e-06, "ci_vs_ctrl": [-6.393072952146084e-05, 7.37865104797425e-05], "sd": 3.50022857311498e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6441126350872265} +{"name": "prop_all", "cut": 23, "nfeat": 28, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7958092803166474, "d_vs_z": 0.00015350292918181463, "d_vs_ctrl": 0.0002638418532376896, "ci_vs_ctrl": [0.00011603082194248937, 0.000432235776237011], "sd": 8.091216243359532e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6479624908242326} diff --git a/dw_lb/results_p4.jsonl b/dw_lb/results_p4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4b81956c1cd95bba9491a1d70c24c921b40f7302 --- /dev/null +++ b/dw_lb/results_p4.jsonl @@ -0,0 +1,2 @@ +{"cut": 25, "ev_lse": 0.6680507246702352, "ev_max": 0.6221356536575582, "ev_ctx": 0.6712369663700499, "ev_ctx_prop": 0.6723044047655211, "glob_ctx": 0.8170725392055592, "glob_ctx_prop": 0.8171975022435236, "glob_z": 0.8172259475835104, "glob_oracle_ev": 0.8939614341668701} +{"cut": 23, "ev_lse": 0.644454956712242, "ev_max": 0.6033170604590983, "ev_ctx": 0.6487983224904563, "ev_ctx_prop": 0.6523494305713399, "glob_ctx": 0.7950750830226788, "glob_ctx_prop": 0.7952474179617948, "glob_z": 0.7956557773874656, "glob_oracle_ev": 0.881737373317231} diff --git a/dw_lb/results_tx.jsonl b/dw_lb/results_tx.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b347b3cdc19cd4cd3e5f39cc95b902a6419ed971 --- /dev/null +++ b/dw_lb/results_tx.jsonl @@ -0,0 +1,18 @@ +{"name": "tx_f1", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172027883512023, "d_vs_ctrl": -1.6709914137047477e-05, "ci": [-0.00014349266344125257, 0.00014587432118149673], "sd": 7.394595909960953e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173499925039057, "d_vs_ctrl": 0.00013049423856636988, "ci": [3.9790617533291826e-05, 0.00023067132830789097], "sd": 5.150322756776621e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173169526524945, "d_vs_ctrl": 9.74543871551381e-05, "ci": [-5.4981787659713624e-05, 0.00021526865679996352], "sd": 7.300359621086734e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172527064819, "d_vs_ctrl": 3.320821656060957e-05, "ci": [-5.4703244326295694e-05, 0.00011614454119170672], "sd": 4.313636553203988e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173167974531333, "d_vs_ctrl": 9.729918779399771e-05, "ci": [-7.801107898108007e-05, 0.00023711497566265008], "sd": 8.903256137410697e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173983206258677, "d_vs_ctrl": 0.00017882236052835232, "ci": [-5.831556720231612e-06, 0.00032584640888283224], "sd": 8.898557768981401e-05, "nfeat": 88} +{"name": "tx_f1", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956233103415669, "d_vs_ctrl": 7.787187815710972e-05, "ci": [-7.479367148836014e-05, 0.00026347507973854004], "sd": 8.39083487483491e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7954944826046069, "d_vs_ctrl": -5.095585880288134e-05, "ci": [-0.00013868587711693414, 4.4620645681803275e-05], "sd": 4.891152042204878e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956304922619755, "d_vs_ctrl": 8.505379856571516e-05, "ci": [-4.9795107496017343e-05, 0.00022870341233730828], "sd": 7.476699867419253e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956183987133403, "d_vs_ctrl": 7.296024993053152e-05, "ci": [-3.4362482472535814e-05, 0.0001534660315574921], "sd": 4.601110414256159e-05, "nfeat": 6} +{"name": "tx_all", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956668950159012, "d_vs_ctrl": 0.00012145655249140574, "ci": [-4.162412313422625e-05, 0.0002827905883446003], "sd": 8.632411276152597e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7958353874293667, "d_vs_ctrl": 0.0002899489659569099, "ci": [0.00012652061304543327, 0.00046809093858551367], "sd": 8.882145961222389e-05, "nfeat": 88} +{"name": "tx_f1", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8171540011704558, "d_vs_ctrl": -4.7271458446407344e-05, "ci": [-0.00019061083887653984, 9.264123771784986e-05], "sd": 8.080654707256493e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173168932756604, "d_vs_ctrl": 0.00011562064675818551, "ci": [3.137035239061392e-05, 0.0002165138368945774], "sd": 4.745692855025363e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173267896762714, "d_vs_ctrl": 0.00012551704736918357, "ci": [-1.1038759591067149e-05, 0.0002785910167285549], "sd": 7.539173816284816e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8172492119677455, "d_vs_ctrl": 4.793933884328361e-05, "ci": [-4.7446992155203425e-05, 0.00013619981497832174], "sd": 4.878073514728382e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.817286679273291, "d_vs_ctrl": 8.540664438883017e-05, "ci": [-9.598281369740669e-05, 0.00025745601599898863], "sd": 8.95598780600043e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173714409684247, "d_vs_ctrl": 0.00017016833952254729, "ci": [9.941956942391419e-06, 0.00032593622615555433], "sd": 8.941576662609524e-05, "nfeat": 88} diff --git a/dw_lb/run_p4.py b/dw_lb/run_p4.py new file mode 100644 index 0000000000000000000000000000000000000000..c620a58280e9452a34287c952f2b4fa7d7d9dc17 --- /dev/null +++ b/dw_lb/run_p4.py @@ -0,0 +1,24 @@ +from lib import * +import sys +out=[] +for c in (25,23): + F=load_cut(c); P=np.load(f'cache_prop_c{c}.npy'); names=json.load(open('prop_names.json')); y,z,u=F['y'],F['z'],F['user'] + n=len(y)//10; Z=z.reshape(n,10); Y=y.reshape(n,10); ey=Y.max(1); eu=u[::10] + lse=np.log(np.exp(Z).sum(1)); sm=np.exp(Z-lse[:,None])/1 ; mx=Z.max(1); P1=P[::10] + def ev(f): return A(ey,f) + ctx=np.column_stack([lse,mx,Z.mean(1),Z.std(1),np.sort(Z,1)[:,-2]]) + def evlgb(X,rounds=150): + o=np.zeros(n); + for tr,te in GroupKFold(5).split(X,ey,eu): + m=lgb.train({**PI,'min_data_in_leaf':200,'num_leaves':7},lgb.Dataset(X[tr],ey[tr]),rounds); o[te]=m.predict(X[te],raw_score=True) + return o + r=dict(cut=c,ev_lse=ev(lse),ev_max=ev(mx)) + o1=evlgb(ctx); o2=evlgb(np.hstack([ctx,P1])); r.update(ev_ctx=ev(o1),ev_ctx_prop=ev(o2)) + # 2-stage global score: logit of P(click_j)=P(ev)*softmax_j + lp=lambda o:(np.log(1/(1+np.exp(-o)))[:,None]+Z-lse[:,None]).ravel() + for nm,o in (('ctx',o1),('ctx_prop',o2)): + s=lp(o); r['glob_'+nm]=A(y,s) + r['glob_z']=A(y,z) + # oracle softmax with perfect event label + r['glob_oracle_ev']=A(y,(np.log(np.clip(np.repeat(ey,10),1e-3,1))+ (Z-lse[:,None]).ravel())) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open('results_p4.jsonl','a').write(json.dumps(r)+'\n') diff --git a/dw_lb/run_prop.log b/dw_lb/run_prop.log new file mode 100644 index 0000000000000000000000000000000000000000..7505d11cf937ebe444b1fc8098d0ccd34b114659 --- /dev/null +++ b/dw_lb/run_prop.log @@ -0,0 +1,6 @@ +25 (439640, 28) +{"name": "null_event_noise", "cut": 25, "nfeat": 1, "base": 0.81723, "control": 0.81722, "test": 0.81716, "d_vs_z": -7e-05, "d_vs_ctrl": -6e-05, "ci_vs_ctrl": [-0.00010993807087290585, -7.890079290281935e-06], "sd": 3e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66566} +{"name": "prop_all", "cut": 25, "nfeat": 28, "base": 0.81723, "control": 0.81722, "test": 0.81742, "d_vs_z": 0.00019, "d_vs_ctrl": 0.0002, "ci_vs_ctrl": [8.425431766705683e-05, 0.00032945002107639954], "sd": 6e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66715} +23 (501550, 28) +{"name": "null_event_noise", "cut": 23, "nfeat": 1, "base": 0.79566, "control": 0.79555, "test": 0.79555, "d_vs_z": -0.00011, "d_vs_ctrl": 0.0, "ci_vs_ctrl": [-6.393072952146084e-05, 7.37865104797425e-05], "sd": 4e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64411} +{"name": "prop_all", "cut": 23, "nfeat": 28, "base": 0.79566, "control": 0.79555, "test": 0.79581, "d_vs_z": 0.00015, "d_vs_ctrl": 0.00026, "ci_vs_ctrl": [0.00011603082194248937, 0.000432235776237011], "sd": 8e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64796} diff --git a/dw_lb/run_prop.py b/dw_lb/run_prop.py new file mode 100644 index 0000000000000000000000000000000000000000..2d8b6790f93480f8613e34d04d64b036ed67cafe --- /dev/null +++ b/dw_lb/run_prop.py @@ -0,0 +1,8 @@ +from lib import * +rng=np.random.RandomState(0) +for c in (25,23): + F=load_cut(c); P=np.load(f'cache_prop_c{c}.npy'); names=json.load(open('prop_names.json')) + print(c,P.shape,flush=True) + noise=np.repeat(rng.randn(len(F['y'])//10,1).astype(np.float32),10,1).reshape(-1,1) + report('null_event_noise',F,noise) + report('prop_all',F,P) diff --git a/dw_lb/run_tx.log b/dw_lb/run_tx.log new file mode 100644 index 0000000000000000000000000000000000000000..2b984ac86b71239e2c321be3a0ee3d9c4959ac52 --- /dev/null +++ b/dw_lb/run_tx.log @@ -0,0 +1,12 @@ +{"name": "tx_f1", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.8172, "d_vs_ctrl": -2e-05, "ci": [-0.00014349266344125257, 0.00014587432118149673], "sd": 7e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.81735, "d_vs_ctrl": 0.00013, "ci": [3.9790617533291826e-05, 0.00023067132830789097], "sd": 5e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.81732, "d_vs_ctrl": 0.0001, "ci": [-5.4981787659713624e-05, 0.00021526865679996352], "sd": 7e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.81725, "d_vs_ctrl": 3e-05, "ci": [-5.4703244326295694e-05, 0.00011614454119170672], "sd": 4e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.81732, "d_vs_ctrl": 0.0001, "ci": [-7.801107898108007e-05, 0.00023711497566265008], "sd": 9e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 0, "base": 0.81723, "control": 0.81722, "test": 0.8174, "d_vs_ctrl": 0.00018, "ci": [-5.831556720231612e-06, 0.00032584640888283224], "sd": 9e-05, "nfeat": 88} +{"name": "tx_f1", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79562, "d_vs_ctrl": 8e-05, "ci": [-7.479367148836014e-05, 0.00026347507973854004], "sd": 8e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79549, "d_vs_ctrl": -5e-05, "ci": [-0.00013868587711693414, 4.4620645681803275e-05], "sd": 5e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79563, "d_vs_ctrl": 9e-05, "ci": [-4.9795107496017343e-05, 0.00022870341233730828], "sd": 7e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79562, "d_vs_ctrl": 7e-05, "ci": [-3.4362482472535814e-05, 0.0001534660315574921], "sd": 5e-05, "nfeat": 6} +{"name": "tx_all", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79567, "d_vs_ctrl": 0.00012, "ci": [-4.162412313422625e-05, 0.0002827905883446003], "sd": 9e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 23, "seed": 0, "base": 0.79566, "control": 0.79555, "test": 0.79584, "d_vs_ctrl": 0.00029, "ci": [0.00012652061304543327, 0.00046809093858551367], "sd": 9e-05, "nfeat": 88} diff --git a/dw_lb/run_tx.py b/dw_lb/run_tx.py new file mode 100644 index 0000000000000000000000000000000000000000..5913f3e99ba0fbb0fb4e59e527302a8c7d110d1e --- /dev/null +++ b/dw_lb/run_tx.py @@ -0,0 +1,14 @@ +from lib import * +import sys +cuts=[int(x) for x in sys.argv[1].split(',')]; seeds=[int(x) for x in sys.argv[2].split(',')] +names=json.load(open('tx_names.json')); fam={f:[i for i,n in enumerate(names) if n.startswith(f)] for f in ['f1','f2','f3','f4']} +sets=dict(f1=fam['f1'],f2=fam['f2'],f3=fam['f3'],f4=fam['f4'],all=list(range(len(names)))) +for c in cuts: + F=load_cut(c); T=np.load(f'cache_tx_c{c}.npy'); P=np.load(f'cache_prop_c{c}.npy'); y,z,u=F['y'],F['z'],F['user'] + for s in seeds: + pc=oof(F['Xb'],y,z,u,s); + for nm,idx in list(sets.items())+[('all+prop',None)]: + X=np.hstack([F['Xb'],T if idx is None else T[:,idx]]+([P] if idx is None else [])) + pt=oof(X,y,z,u,s); ci,sd=boot_delta(y,pc,pt,None,nb=150) + r=dict(name='tx_'+nm,cut=c,seed=s,base=A(y,z),control=A(y,pc),test=A(y,pt),d_vs_ctrl=A(y,pt)-A(y,pc),ci=[float(ci[0]),float(ci[1])],sd=float(sd),nfeat=X.shape[1]-F['Xb'].shape[1]) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open('results_tx.jsonl','a').write(json.dumps(r)+'\n') diff --git a/dw_lb/run_tx_s1.log b/dw_lb/run_tx_s1.log new file mode 100644 index 0000000000000000000000000000000000000000..540453494e085d1ce14ad392e9e608d5a032019f --- /dev/null +++ b/dw_lb/run_tx_s1.log @@ -0,0 +1,6 @@ +{"name": "tx_f1", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81715, "d_vs_ctrl": -5e-05, "ci": [-0.00019061083887653984, 9.264123771784986e-05], "sd": 8e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81732, "d_vs_ctrl": 0.00012, "ci": [3.137035239061392e-05, 0.0002165138368945774], "sd": 5e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81733, "d_vs_ctrl": 0.00013, "ci": [-1.1038759591067149e-05, 0.0002785910167285549], "sd": 8e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81725, "d_vs_ctrl": 5e-05, "ci": [-4.7446992155203425e-05, 0.00013619981497832174], "sd": 5e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81729, "d_vs_ctrl": 9e-05, "ci": [-9.598281369740669e-05, 0.00025745601599898863], "sd": 9e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 1, "base": 0.81723, "control": 0.8172, "test": 0.81737, "d_vs_ctrl": 0.00017, "ci": [9.941956942391419e-06, 0.00032593622615555433], "sd": 9e-05, "nfeat": 88} diff --git a/dw_lb/sm25.log b/dw_lb/sm25.log new file mode 100644 index 0000000000000000000000000000000000000000..85a50762a04bd51dc49f4d08dac50ba818f2dde1 --- /dev/null +++ b/dw_lb/sm25.log @@ -0,0 +1,6 @@ +c25 ep0 loss 1.9891 val AUC logP 0.60193 t=6s +c25 ep1 loss 1.9703 val AUC logP 0.59845 t=10s +c25 ep2 loss 1.8355 val AUC logP 0.59332 t=13s +c25 ep3 loss 1.6177 val AUC logP 0.58636 t=17s +c25 ep4 loss 1.3677 val AUC logP 0.58078 t=20s +c25 ep5 loss 1.0478 val AUC logP 0.57815 t=24s diff --git a/dw_lb/tx25.log b/dw_lb/tx25.log new file mode 100644 index 0000000000000000000000000000000000000000..7a5fe4f83d0db5ac717a74eb632e55c4527b7437 --- /dev/null +++ b/dw_lb/tx25.log @@ -0,0 +1,2 @@ +25 (439640, 60) +23 (501550, 60) diff --git a/dw_lb/tx_names.json b/dw_lb/tx_names.json new file mode 100644 index 0000000000000000000000000000000000000000..d1e15253c584935d84b88ba3200edca3659227cc --- /dev/null +++ b/dw_lb/tx_names.json @@ -0,0 +1 @@ +["f1_cnt", "f1_cnt_h1", "f1_cnt_h6", "f1_cnt_h6_raw", "f1_cnt_h24", "f1_late_exp", "f1_late_ctr", "f1_ratio_late", "f1_ratio_all", "f1_exp_x_ctr", "f1_cnt_z", "f1_cnt_dm", "f1_cnt_rk", "f1_cnt_h6_z", "f1_cnt_h6_dm", "f1_cnt_h6_rk", "f1_ratio_late_z", "f1_ratio_late_dm", "f1_ratio_late_rk", "f1_slate_mean", "f3_hpop", "f3_hpop_l3", "f3_hpop_l1", "f3_hpost", "f3_hrecent_6h", "f3_hnext_6h", "f3_hrecent_24h", "f3_hnext_24h", "f3_own_cnt", "f3_hpop_excl_own", "f3_hpop_z", "f3_hpop_dm", "f3_hpop_rk", "f3_hpost_z", "f3_hpost_dm", "f3_hpost_rk", "f3_hrecent_24h_z", "f3_hrecent_24h_dm", "f3_hrecent_24h_rk", "f3_hpost_vs_train_ctr", "f2_cooc_norm", "f2_cooc_raw", "f2_lastshare_n", "f2_lastshare", "f2_l3share", "f2_cooc_norm_z", "f2_cooc_norm_dm", "f2_cooc_norm_rk", "f2_lastshare_z", "f2_lastshare_dm", "f2_lastshare_rk", "f2_l3share_z", "f2_l3share_dm", "f2_l3share_rk", "f4_trel_h", "f4_trank", "f4_n_same_h", "f4_n_6h", "f4_n_before_frac", "f4_hist_n"] \ No newline at end of file diff --git a/dw_levers/blend_div.py b/dw_levers/blend_div.py new file mode 100644 index 0000000000000000000000000000000000000000..ce3ac0f64fbcaa4febce712c40f58c901f0d21c5 --- /dev/null +++ b/dw_levers/blend_div.py @@ -0,0 +1,19 @@ +import sys, numpy as np, polars as pl, glob +sys.path.insert(0,'/workspace/huawei/dw_levers') +from common import z12, A, H +from scipy.stats import rankdata +for c in (25,23): + y=pl.read_parquet(f'{H}/v10/feat_v3/val{c}.parquet',columns=['clicked']).to_series().to_numpy().astype(int) + z=z12(c); print(f'== C{c} z12 {A(y,z):.5f}') + M={} + for f in sorted(glob.glob(f'{H}/v12/ens/*_c{c}_s*.npy')): + a=np.load(f); nm=f.split('/')[-1][:-4] + if a.ndim==1: a=a[:,None] + for k in range(a.shape[1]): M[f'{nm}[{k}]']=a[:,k] + for nm,a in M.items(): + if np.isnan(a).any() or len(a)!=len(y): continue + r=A(y,a); + if r<0.7: continue + # incremental: best w in z + w*zscore(a) + zs=(a-a.mean())/a.std(); best=max((A(y,z+w*zs*z.std()),w) for w in (0,0.05,0.1,0.2,0.4)) + print(f' {nm:18s} AUC {r:.5f} corr(z) {np.corrcoef(z,a)[0,1]:.3f} best blend {best[0]-A(y,z):+.5f} w={best[1]}') diff --git a/dw_levers/cfg1.txt b/dw_levers/cfg1.txt new file mode 100644 index 0000000000000000000000000000000000000000..06185d2b57e65d5fc15be9c850822ab790e7bcac --- /dev/null +++ b/dw_levers/cfg1.txt @@ -0,0 +1,6 @@ +{"d":32,"ep":3,"lr":0.001} +{"d":32,"ep":3,"lr":0.001,"l2":0.01} +{"d":32,"ep":3,"lr":0.001,"l2":0.1} +{"d":32,"ep":3,"lr":0.001,"hist_drop":0.3,"l2":0.01} +{"d":32,"ep":3,"lr":0.001,"att":true,"l2":0.01} +{"d":64,"ep":3,"lr":0.001,"l2":0.03} diff --git a/dw_levers/cfg2.txt b/dw_levers/cfg2.txt new file mode 100644 index 0000000000000000000000000000000000000000..c350b9d548f3844117e04cda55d942338c672d44 --- /dev/null +++ b/dw_levers/cfg2.txt @@ -0,0 +1,3 @@ +{"d":32,"ep":6,"lr":0.003,"elr":0.0} +{"d":32,"ep":6,"lr":0.003,"elr":0.05} +{"d":32,"ep":6,"lr":0.003,"elr":0.15,"hist_drop":0.2} diff --git a/dw_levers/choice.py b/dw_levers/choice.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a4af38901720063ffa28f8dcbb1b8fdb45f890 --- /dev/null +++ b/dw_levers/choice.py @@ -0,0 +1,105 @@ +"""Choice model with outside option (CPU PyTorch). P(click i | slate) = exp(s_i)/(exp(o)+sum_j exp(s_j)), s_i=+b_i, +u = sum_h w(age_h) * Eh[h] / n^a (+ optional free user vec). Trained on train events with ts model ; score(model,D,ev_idx) -> (n,10) log-prob (and (n,) log p_none)""" +import numpy as np, torch, torch.nn as nn, time, sys +sys.path.insert(0,'/workspace/huawei/dw_levers') +from dwcommon import * +from common import A +torch.set_num_threads(4) +HL=20 +def build_hist(D): + if hasattr(D,'H'): return + n=D.NE; H=np.full((n,HL),D.NI,np.int32); G=np.zeros((n,HL),np.int8) + rows=np.repeat(np.arange(n),D.Hlen); cols=np.arange(len(D.HI))-np.repeat(D.OFF[:-1],D.Hlen) + H[rows,cols]=D.HI; gap=np.maximum(D.ETS[rows]-D.HT,60) + G[rows,cols]=1+np.clip(np.log2(gap/60),0,19).astype(np.int8) + D.H=H; D.G=G +class CM(nn.Module): + def __init__(s,NI,NU,V0,d,free_user,att): + super().__init__(); s.NI=NI + s.E=nn.Embedding(NI+1,d,padding_idx=NI); s.Eh=nn.Embedding(NI+1,d,padding_idx=NI); s.b=nn.Embedding(NI+1,1,padding_idx=NI) + with torch.no_grad(): + k=V0.shape[1]; E0=torch.randn(NI+1,d)*0.01; E0[:NI,:k]=torch.from_numpy(V0); E0[NI]=0 + s.E.weight.copy_(E0); s.Eh.weight.copy_(E0); s.b.weight.zero_() + s.gw=nn.Embedding(HL+2,1); nn.init.zeros_(s.gw.weight) # age weight logits + s.scale=nn.Parameter(torch.tensor(3.0)); s.alpha=nn.Parameter(torch.tensor(0.5)) + s.free=nn.Embedding(NU,d) if free_user else None + if free_user: nn.init.zeros_(s.free.weight) + s.att=att + if att: s.Wq=nn.Linear(d,d,bias=False); nn.init.eye_(s.Wq.weight) + s.out=nn.Sequential(nn.Linear(4,16),nn.ReLU(),nn.Linear(16,1)) # outside logit from user/ctx stats + s.ob=nn.Embedding(NU,1) if free_user else None + if free_user: nn.init.zeros_(s.ob.weight) + def forward(s,H,G,cand,u,ctx): + valid=(H!=s.NI).float(); nh=valid.sum(1,keepdim=True) + eh=s.Eh(H); w=torch.sigmoid(s.gw(G.long()).squeeze(-1)+1.0)*valid # (B,T) + q=s.E(cand) # (B,10,d) + if s.att: + a=torch.einsum('bcd,btd->bct',s.Wq(q),eh)/eh.shape[-1]**0.5*s.scale; a=a.masked_fill(valid[:,None,:]==0,-1e9) + wt=torch.softmax(a,-1)*(nh>0).float()[:,:,None]; ur=torch.einsum('bct,btd->bcd',wt*w[:,None,:],eh) # per-candidate user rep + sc=(ur*q).sum(-1)*s.scale/ (w.sum(1,keepdim=True)+1e-3).clamp(min=1)**0.0 + else: + ur=(eh*w[:,:,None]).sum(1)/(nh+1.0)**s.alpha # (B,d) + if s.free is not None: ur=ur+s.free(u) + sc=torch.einsum('bd,bcd->bc',ur,q)*s.scale + sc=sc+s.b(cand).squeeze(-1) + f=torch.cat([torch.log1p(nh),ctx],1) # ctx: (B,3) + o=s.out(f)+(s.ob(u) if s.ob is not None else 0) + return sc,o.squeeze(-1) +def ctx_feats(D,ev): + tb=((D.ETS[ev]-J20)%DAY)/DAY; dt=(((D.ETS[ev]-J20)//DAY))%7/7.0 + nlast=np.where(D.Hlen[ev]>0,D.G[ev].max(1),0)/20.0 # gap-bucket of the oldest? (crude recency) -> use min nonzero below + G=D.G[ev].astype(np.float32); G[G==0]=99; rec=np.where(D.Hlen[ev]>0,G.min(1),0)/20.0 + return np.stack([tb,dt,rec],1).astype(np.float32) +def fit_choice(D,X,cfg,log_every=1,val=None): + build_hist(D); t0=time.time(); torch.manual_seed(cfg.get('seed',0)); np.random.seed(cfg.get('seed',0)) + nX=int(np.searchsorted(D.ETS,X)); idx=np.arange(nX) + d=cfg.get('d',32); S=D.snap(X); V0,_=click_svd(S,D,k=min(d,cfg.get('k',32))); log('svd done') + V0=V0*cfg.get('v0scale',1.0) + m=CM(D.NI,D.NU,V0,d,cfg.get('free_user',False),cfg.get('att',False)) + tH=torch.from_numpy(D.H.astype(np.int64)); tG=torch.from_numpy(D.G.astype(np.int64)); tC=torch.from_numpy(D.RI.astype(np.int64)); tU=torch.from_numpy(D.EU.astype(np.int64)) + ctx=torch.from_numpy(ctx_feats(D,np.arange(D.NE))) + yidx=np.where(D.Y.sum(1)>0,D.Y.argmax(1),10); tY=torch.from_numpy(yidx.astype(np.int64)) + opt=torch.optim.Adam([{'params':[p for n_,p in m.named_parameters() if n_.split('.')[0] not in ('free','ob','E','Eh','b')]}, + {'params':list(m.E.parameters())+list(m.Eh.parameters())+list(m.b.parameters()),'weight_decay':0.0,'lr':cfg.get('lr',2e-3)*cfg.get('elr',1.0)}, + {'params':(list(m.free.parameters())+list(m.ob.parameters())) if m.free is not None else [],'weight_decay':0.0}],lr=cfg.get('lr',2e-3)) + bs=cfg.get('bs',512); wd=cfg.get('l2',0.0); fl2=cfg.get('free_l2',1e-4); ep=cfg.get('ep',4); hid=cfg.get('hist_drop',0.0) + # held-out tail of fit window for early stopping (last 5% of events by time) + ntr=int(nX*0.95); hold=np.arange(ntr,nX); idx=np.arange(ntr) + best=(1e9,None) + for e in range(ep): + m.train(); perm=torch.from_numpy(np.random.permutation(idx)); tl=0; nb=0 + for b in range(0,len(perm),bs): + j=perm[b:b+bs]; H=tH[j] + if hid>0: H=torch.where(torch.rand(H.shape)0: + loss=loss+wd*((m.E(tC[j])**2).sum(-1).mean()+(m.Eh(H)**2).sum(-1).mean()) + if m.free is not None and fl2>0: loss=loss+fl2*(m.free(tU[j])**2).sum(-1).mean() + opt.zero_grad(); loss.backward(); opt.step(); tl+=loss.item(); nb+=1 + m.eval(); hl=eval_loss(m,D,hold,tH,tG,tC,tU,ctx,tY); msg=f' ep{e} train {tl/nb:.4f} hold {hl:.4f} {time.time()-t0:.0f}s' + if val is not None: + lp,_=score(m,D,val['e'][::10],tH,tG,tC,tU,ctx); lp=to_val(lp,val['col']); msg+=f" valAUC {A(val['y'],lp.ravel()):.5f}" + log(msg) + if hl need only negatives + negs_c=Z[ce][Y[ce]==0] + from scipy.stats import rankdata + def auc2(p,q): + a=np.concatenate([p,q]); r_=rankdata(a); return (r_[:len(p)].sum()-len(p)*(len(p)+1)/2)/(len(p)*len(q)) + r['auc_pos_vs_negNoClickEv']=auc2(pos_s,negs_nc); r['auc_pos_vs_negClickEv(incl same)']=auc2(pos_s,negs_c) + r['n_neg_noclick']=len(negs_nc); r['n_neg_click']=len(negs_c) + # oracles + r['oracle_event_label']=A(y,np.repeat(yev,10)*100+z) # perfect any-click, model within + r['oracle_event_label_only']=A(y,np.repeat(yev,10)) + r['oracle_within']=A(y,np.where(y==1,1,0)*100+ (np.repeat(lse,10))*0 + np.repeat(lse,10)*0 + z*0+ np.repeat(lse,10)) # perfect within-ranking (clicked row top of event), event score lse as before + # perfect within, event score = softmax mass: row score = lse + big*y? use lse + y*delta (delta modest = within gap) + for dl in (1,2,3): + r[f'oracle_within_delta{dl}']=A(y,np.repeat(lse,10)+dl*y*(np.repeat(yev,10))) + # oracle on no-click events: those would be known as no-click, push down by k + for k in (0.5,1,2): + r[f'oracle_noclick_shift{k}']=A(y,z-k*np.repeat(1-yev,10)) + # score mixing: z + a*lse + for a in (-0.5,-0.25,0.25,0.5): + r[f'z+{a}lse']=A(y,z+a*np.repeat(lse,10)) + out[c]=r; print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True) +json.dump(out,open('decomp.json','w')) diff --git a/dw_levers/dwcommon.py b/dw_levers/dwcommon.py new file mode 100644 index 0000000000000000000000000000000000000000..0f07baf375ee6ca9e52ed272e247e2434fe04506 --- /dev/null +++ b/dw_levers/dwcommon.py @@ -0,0 +1,72 @@ +"""Shared loader for dw_signals. READ-ONLY on everything outside /workspace/huawei/dw_signals. +Snapshot(C): everything known from train events with ts < C (rows+labels of those events, plus click log reconstructed from their histories). +Val(C): each user's first event with ts in [C, Jan27) (rows in the same order as v10/feat_v3/val{C}.parquet == v11/ens/*_c{C}_*.npy rows).""" +import numpy as np, polars as pl, time, gc +from scipy.sparse import csr_matrix +from scipy.sparse.linalg import svds +L='/workspace/huawei/lite'; DAY=86400; J20=1737331200 +def Jd(d): return J20+(d-20)*DAY +T0=time.time() +def log(*a): print(f'[{time.time()-T0:6.0f}s]',*a,flush=True) + +class Data: + def __init__(s): + ev=pl.read_parquet(f'{L}/train_events.parquet').sort('ts').with_row_index('e') + rows=pl.read_parquet(f'{L}/train_rows.parquet').join(ev.select('eid','e'),on='eid').sort(['e','item']) + s.NE=ev.height; assert rows.height==s.NE*10 + s.ETS=ev['ts'].to_numpy(); s.EID=ev['eid'].to_numpy() + uu,s.EU=np.unique(ev['user'].to_numpy(),return_inverse=True); s.EU=s.EU.astype(np.int32); s.NU=len(uu); s.uu=uu + s.Hlen=ev['h_items'].list.len().to_numpy().astype(np.int64) + s.OFF=np.zeros(s.NE+1,np.int64); s.OFF[1:]=np.cumsum(s.Hlen) + hi=ev['h_items'].explode().drop_nulls().to_numpy(); s.HT=ev['h_ts'].explode().drop_nulls().to_numpy().astype(np.int64) + rr=rows['item'].to_numpy() + s.items=np.unique(np.concatenate([np.unique(rr),np.unique(hi)])); s.NI=len(s.items) + s.HI=np.searchsorted(s.items,hi).astype(np.int32); s.RI=np.searchsorted(s.items,rr).astype(np.int32).reshape(s.NE,10) + s.Y=rows['clicked'].to_numpy().astype(np.int8).reshape(s.NE,10) + s.rowid=rows['rowid'].to_numpy().reshape(s.NE,10) + s.eid2e={} # not needed + s.ev=ev.select('e','eid','user','ts'); s.rows_rowid=rows.select('rowid','e','item') + del ev,rows; gc.collect() + def val(s,c): + """returns dict with aligned arrays for val cut c: rowid, y, e (event idx), j(row pos in event), item idx, user idx, ts""" + v=pl.read_parquet(f'/workspace/huawei/v10/feat_v3/val{c}.parquet',columns=['rowid','clicked']) + m=v.join(s.rows_rowid,on='rowid',how='left',maintain_order='left') + e=m['e'].to_numpy().astype(np.int64); it=np.searchsorted(s.items,m['item'].to_numpy()).astype(np.int32) + assert (e[::10]==e.reshape(-1,10)[:,0]).all() and (e.reshape(-1,10)==e.reshape(-1,10)[:,:1]).all() + return dict(rowid=v['rowid'].to_numpy(),y=v['clicked'].to_numpy().astype(np.int8),e=e,item=it,user=s.EU[e],ts=s.ETS[e]) + def snap(s,X): + """click log (user,item,ts) from events with ts>20; tm=key&((1<<20)-1) + return dict(nX=nX,cu=(ui//s.NI).astype(np.int32),ci=(ui%s.NI).astype(np.int32),ct=tm*60+Jd(1)) + +def click_svd(S,D,k=32): + """item/user taste vectors from dedup click log (binary, 1/sqrt((nu+1)(ni+1)) scaling). returns item vec (V*s) L2-normed, user vec""" + u,i=S['cu'],S['ci']; key=np.unique(u.astype(np.int64)*D.NI+i); uu=(key//D.NI).astype(np.int32); ii=(key%D.NI).astype(np.int32) + nu=np.bincount(uu,minlength=D.NU).astype(np.float32); ni=np.bincount(ii,minlength=D.NI).astype(np.float32) + M=csr_matrix((1/np.sqrt((nu[uu]+1)*(ni[ii]+1)),(uu,ii)),shape=(D.NU,D.NI),dtype=np.float32) + U,sv,Vt=svds(M,k=k,random_state=0) + V=(Vt.T*sv); V/= np.linalg.norm(V,axis=1,keepdims=True)+1e-9 + return V.astype(np.float32),(U*sv).astype(np.float32) +def norm(x): return x/(np.linalg.norm(x,axis=-1,keepdims=True)+1e-9) +def event_view(a): return a.reshape(-1,10) +def within_event(x): + """z-score, demeaned, rank (0..9 ascending) within the 10-row events; x aligned with val rows""" + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + rk=m.argsort(1).argsort(1).astype(np.float32) + return ((m-mu)/sd).ravel(),(m-mu).ravel(),rk.ravel() + +V11='/workspace/huawei/v11' +def v11_pred(c): + a=np.load(f'{V11}/ens/sCbig_c{c}_s0.npy')[:,1]; b=np.load(f'{V11}/ens/sC_s_c{c}_s0.npy')[:,1] + m=np.mean([np.load(f'{V11}/ens/mlp2_c{c}_s{k}.npy')[:,3] for k in range(3)],0) + return (a+b+2*m)/4 +def segs(c): + m=pl.read_parquet(f'/workspace/huawei/v10/feat_v3/val{c}.parquet',columns=['us_cold','us_nohist']) + st=pl.read_parquet(f'/workspace/huawei/v10/feat_tmB2/val{c}.parquet')['tmR_stale_h'].to_numpy() + return dict(nohist=m['us_nohist'].to_numpy(),cold=m['us_cold'].to_numpy(),stale=st) diff --git a/dw_levers/findings.md b/dw_levers/findings.md new file mode 100644 index 0000000000000000000000000000000000000000..4622067df308a1ab22d3735a3d0b9903b9917710 --- /dev/null +++ b/dw_levers/findings.md @@ -0,0 +1,35 @@ +# dw_levers findings (Europe/Dublin, 2026-10-28) +Baseline = v12 z12 blend (C25 0.81723, C23 0.79566). All numbers on val rows (user's first event after cutoff). + +## AUC decomposition (decomp.json) +| | C25 | C23 | +|---|---|---| +| Global AUC | 0.8172 | 0.7957 | +| Within-event pooled AUC | 0.8258 | 0.8043 | +| Event "any click" AUC of 1-prod(1-p) | 0.673 | 0.650 | +| Oracle event label + model within-event | 0.891 | 0.879 | +| Within-event pair share of all pairs | 2e-5 | 2e-5 | +Global AUC is almost entirely between-event, so it is set by cross-event calibration (a mix of item quality + event-level click propensity). Headroom sim (headroom_sim.json): event AUC 0.673->0.709 = +0.003, ->0.758 = +0.0075, ->0.80 ≈ +0.012. A +0.012 gain from the event lever alone needs event AUC ≈0.80 (from 0.67), not realistic: v13 exp1 event residual models were negative, my choice-model "no-click" head has event AUC 0.62 (<0.673). + +## Ranked levers (ΔAUC over z12, C25) +| # | lever | result | +|---|---|---| +| 1 | Event-level model (headroom + v13 + choice outside option) | long shot, needs event AUC +0.13; every attempt ≤0 | +| 2 | Choice model w/ outside option, PyTorch end-to-end (choice.py) | standalone 0.778 (lr1e-3, 1 epoch; best-by-holdout), blend with z: -0.0001 (w=.1) to -0.003 (w=.4); LGBM residual stacking with lp/slate-rank/outside logit vs control with only z-derived feats: +0.00008 [95% CI +0.00002,+0.00015] (event bootstrap) = negligible | +| 3 | Two-tower/seq/LightGCN | Not built separately: choice model is effectively this (SVD-init item/user-hist embeddings + softmax) and reached only 0.78; frozen SVD embeddings 0.753; attention 0.768; hist dropout / l2 / lr sweeps (sweep_choice.py, choice_sweep.jsonl) all 0.75-0.778 and overfit after 1 epoch (holdout loss min at epoch 0). Earlier DIN 0.534. Not worth blending | +| 4 | Rank/linear blend diversity (blend_div.py) | all 44 existing ens members (dw_s, dwbig, dwmlp, mlp2, sC, sCbig × seeds) correlate .97-.995 with z12; best incremental blend +0.00007 (dwbig seed1). Ensemble is saturated | +| 5 | Error analysis / cross-segment recalibration (seg.py, recal.py) | Per-segment Platt recalibration by us_hlen, it_imp, us_nev (+ crosses), user-grouped CV: Δ -0.0003..+0.00006 → score already calibrated across segments, no headroom | + +Stale rows: `tm_stale` in val parquet is not hours (all >48), so C23 stale slice was not separable via it; no model in the ensemble or the choice model is better on any slice—ens members all correlated ≥.97. + +## Where the weakness is (AUC within segment) +C25: hist len ≤6: 0.755 (18% rows) vs hist 17-20: 0.836 (49%); users with ≤2 events 0.767 vs ≥10 events 0.84; low-impression items (it_imp<61) 0.78 vs >237 0.84. C23: hlen≤5 0.730 (21%), us_nev≤1 0.751 (24%), it_imp≤36 0.757. So cold users/cold items are the weak spots — information-limited (no history), not model-limited; every learned model fails there too. + +## Honest notes / failures +- First choice-model run was invalid (val AUC 0.496): scores were in D.RI order (rows sorted by item), val labels in parquet order. Fixed with align()/to_val() in choice.py. +- Choice model overfits: 50k×32 free item embeddings on ~490k events; best-by-holdout is epoch 0-1. l2 / hist-dropout / low emb-lr don't get above 0.778 (SVD frozen 0.753). +- Everything ≥ plateau; no lever found with ≥+0.001. Gap to 0.83 is not closable by blend/stack/neural diversity on the current information set. + +## Files +common.py, dwcommon.py, decomp.py/json, headroom_sim.py/json, choice.py (CM, fit_choice, score, align, to_val), run_choice.py, sweep_choice.py + cfg1/2.txt + choice_sweep.jsonl, stack_choice.py (LGBM residual stacking test), blend_div.py, seg.py, recal.py, logs/. +Run e.g.: /workspace/venv/bin/python stack_choice.py 25 '{"d":32,"ep":1,"lr":0.001}' ep1 diff --git a/dw_levers/headroom_sim.json b/dw_levers/headroom_sim.json new file mode 100644 index 0000000000000000000000000000000000000000..aab72e374b7207b5e10d8b3be4c66f00c933dbf1 --- /dev/null +++ b/dw_levers/headroom_sim.json @@ -0,0 +1 @@ +{"25": {"base_global": 0.8172259475835104, "base_event": 0.672879467007552, "rows": [{"g": 0.0, "event_auc": 0.6729, "global_auc": 0.81723, "delta": 0.0}, {"g": 0.1, "event_auc": 0.675, "global_auc": 0.81738, "delta": 0.00015}, {"g": 0.2, "event_auc": 0.6801, "global_auc": 0.8178, "delta": 0.00057}, {"g": 0.3, "event_auc": 0.6858, "global_auc": 0.81828, "delta": 0.00106}, {"g": 0.5, "event_auc": 0.7093, "global_auc": 0.82044, "delta": 0.00321}, {"g": 0.8, "event_auc": 0.7583, "global_auc": 0.82473, "delta": 0.0075}, {"g": 1.2, "event_auc": 0.8279, "global_auc": 0.83223, "delta": 0.015}]}, "23": {"base_global": 0.7956557773874656, "base_event": 0.6498087959114907, "rows": [{"g": 0.0, "event_auc": 0.6498, "global_auc": 0.79566, "delta": 0.0}, {"g": 0.1, "event_auc": 0.6515, "global_auc": 0.79581, "delta": 0.00015}, {"g": 0.2, "event_auc": 0.6599, "global_auc": 0.79636, "delta": 0.0007}, {"g": 0.3, "event_auc": 0.6688, "global_auc": 0.79701, "delta": 0.00136}, {"g": 0.5, "event_auc": 0.6956, "global_auc": 0.79926, "delta": 0.0036}, {"g": 0.8, "event_auc": 0.7517, "global_auc": 0.80431, "delta": 0.00865}, {"g": 1.2, "event_auc": 0.8219, "global_auc": 0.81275, "delta": 0.01709}]}} \ No newline at end of file diff --git a/dw_levers/headroom_sim.py b/dw_levers/headroom_sim.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbad30a7f3dc79890ee1fcf3c8d5180e78ac2dc --- /dev/null +++ b/dw_levers/headroom_sim.py @@ -0,0 +1,13 @@ +import numpy as np, json +from common import * +rng=np.random.default_rng(0); out={} +for c in (25,23): + m=meta(c); y=m['clicked'].to_numpy(); z=z12(c); n=len(y)//10; Z=z.reshape(n,10); yev=y.reshape(n,10).max(1) + pany=1-np.prod(1-1/(1+np.exp(-Z)),1); e0=np.log(pany/(1-pany)); base_ev=A(yev,e0); base=A(y,z); res=[] + for g in (0.0,0.1,0.2,0.3,0.5,0.8,1.2): + s=(yev-.5)*g+rng.normal(size=n); llr=g*s # Bayes update for N(+-g/2,1) side-information + # row logit: p_row=q*softmax -> shifting event log-odds by llr approx shifts row logit by llr*(1-q) + q=pany[:,None]; zz=(Z+llr[:,None]*(1-q)).ravel() + res.append(dict(g=g,event_auc=round(A(yev,e0+llr),4),global_auc=round(A(y,zz),5),delta=round(A(y,zz)-base,5))) + out[c]=dict(base_global=base,base_event=base_ev,rows=res); print(c,base_ev,json.dumps(res),flush=True) +json.dump(out,open('headroom_sim.json','w')) diff --git a/dw_levers/logs/blend_div.log b/dw_levers/logs/blend_div.log new file mode 100644 index 0000000000000000000000000000000000000000..ba11f55b3d7b346e2b7eb78e601e827c86d87522 --- /dev/null +++ b/dw_levers/logs/blend_div.log @@ -0,0 +1,90 @@ +== C25 z12 0.81723 + dw_s_c25_s0[0] AUC 0.81589 corr(z) 0.994 best blend +0.00000 w=0 + dw_s_c25_s0[1] AUC 0.81595 corr(z) 0.995 best blend +0.00000 w=0 + dw_s_c25_s1[0] AUC 0.81610 corr(z) 0.993 best blend +0.00001 w=0.05 + dw_s_c25_s1[1] AUC 0.81621 corr(z) 0.993 best blend +0.00002 w=0.1 + dwbig_c25_s0[0] AUC 0.81647 corr(z) 0.995 best blend +0.00001 w=0.1 + dwbig_c25_s0[1] AUC 0.81658 corr(z) 0.995 best blend +0.00003 w=0.2 + dwbig_c25_s0[2] AUC 0.81654 corr(z) 0.995 best blend +0.00003 w=0.2 + dwbig_c25_s1[0] AUC 0.81651 corr(z) 0.994 best blend +0.00003 w=0.2 + dwbig_c25_s1[1] AUC 0.81652 corr(z) 0.995 best blend +0.00004 w=0.2 + dwbig_c25_s1[2] AUC 0.81651 corr(z) 0.994 best blend +0.00005 w=0.2 + dwmlp_c25_s0[0] AUC 0.81185 corr(z) 0.976 best blend +0.00000 w=0 + dwmlp_c25_s0[1] AUC 0.81507 corr(z) 0.984 best blend +0.00000 w=0 + dwmlp_c25_s0[2] AUC 0.81588 corr(z) 0.993 best blend +0.00000 w=0.05 + dwmlp_c25_s0[3] AUC 0.81637 corr(z) 0.996 best blend +0.00000 w=0 + dwmlp_c25_s1[0] AUC 0.81144 corr(z) 0.978 best blend +0.00000 w=0 + dwmlp_c25_s1[1] AUC 0.81440 corr(z) 0.987 best blend +0.00000 w=0 + dwmlp_c25_s1[2] AUC 0.81592 corr(z) 0.994 best blend +0.00000 w=0 + dwmlp_c25_s1[3] AUC 0.81628 corr(z) 0.995 best blend +0.00000 w=0 + dwmlp_c25_s2[0] AUC 0.81295 corr(z) 0.984 best blend +0.00000 w=0 + dwmlp_c25_s2[1] AUC 0.81485 corr(z) 0.990 best blend +0.00000 w=0 + dwmlp_c25_s2[2] AUC 0.81584 corr(z) 0.992 best blend +0.00000 w=0.05 + dwmlp_c25_s2[3] AUC 0.81632 corr(z) 0.995 best blend +0.00000 w=0 + mlp2_c25_s0[0] AUC 0.81056 corr(z) 0.975 best blend +0.00000 w=0 + mlp2_c25_s0[1] AUC 0.81289 corr(z) 0.983 best blend +0.00000 w=0 + mlp2_c25_s0[2] AUC 0.81420 corr(z) 0.989 best blend +0.00000 w=0 + mlp2_c25_s0[3] AUC 0.81456 corr(z) 0.990 best blend +0.00000 w=0 + mlp2_c25_s1[0] AUC 0.81214 corr(z) 0.976 best blend +0.00000 w=0 + mlp2_c25_s1[1] AUC 0.81286 corr(z) 0.982 best blend +0.00000 w=0 + mlp2_c25_s1[2] AUC 0.81365 corr(z) 0.988 best blend +0.00000 w=0 + mlp2_c25_s1[3] AUC 0.81447 corr(z) 0.990 best blend +0.00000 w=0 + mlp2_c25_s2[0] AUC 0.80838 corr(z) 0.969 best blend +0.00000 w=0 + mlp2_c25_s2[1] AUC 0.81322 corr(z) 0.984 best blend +0.00000 w=0 + mlp2_c25_s2[2] AUC 0.81430 corr(z) 0.987 best blend +0.00000 w=0.05 + mlp2_c25_s2[3] AUC 0.81456 corr(z) 0.990 best blend +0.00000 w=0 + sC_s_c25_s0[0] AUC 0.81408 corr(z) 0.988 best blend +0.00000 w=0 + sC_s_c25_s0[1] AUC 0.81402 corr(z) 0.988 best blend +0.00000 w=0 + sC_s_c25_s1[0] AUC 0.81425 corr(z) 0.988 best blend +0.00000 w=0 + sC_s_c25_s1[1] AUC 0.81425 corr(z) 0.989 best blend +0.00000 w=0 + sCbig_c25_s0[0] AUC 0.81467 corr(z) 0.989 best blend +0.00000 w=0 + sCbig_c25_s0[1] AUC 0.81460 corr(z) 0.989 best blend +0.00000 w=0 + sCbig_c25_s0[2] AUC 0.81456 corr(z) 0.989 best blend +0.00000 w=0.05 + sCbig_c25_s1[0] AUC 0.81461 corr(z) 0.989 best blend +0.00000 w=0 + sCbig_c25_s1[1] AUC 0.81464 corr(z) 0.989 best blend +0.00000 w=0.05 + sCbig_c25_s1[2] AUC 0.81466 corr(z) 0.989 best blend +0.00001 w=0.05 +== C23 z12 0.79566 + dw_s_c23_s0[0] AUC 0.79430 corr(z) 0.992 best blend +0.00000 w=0 + dw_s_c23_s0[1] AUC 0.79415 corr(z) 0.992 best blend +0.00000 w=0 + dw_s_c23_s1[0] AUC 0.79413 corr(z) 0.990 best blend +0.00001 w=0.05 + dw_s_c23_s1[1] AUC 0.79399 corr(z) 0.990 best blend +0.00001 w=0.05 + dwbig_c23_s0[0] AUC 0.79466 corr(z) 0.992 best blend +0.00001 w=0.1 + dwbig_c23_s0[1] AUC 0.79460 corr(z) 0.992 best blend +0.00002 w=0.1 + dwbig_c23_s0[2] AUC 0.79446 corr(z) 0.992 best blend +0.00002 w=0.1 + dwbig_c23_s1[0] AUC 0.79470 corr(z) 0.991 best blend +0.00005 w=0.2 + dwbig_c23_s1[1] AUC 0.79458 corr(z) 0.991 best blend +0.00005 w=0.2 + dwbig_c23_s1[2] AUC 0.79450 corr(z) 0.991 best blend +0.00007 w=0.2 + dwmlp_c23_s0[0] AUC 0.78745 corr(z) 0.968 best blend +0.00000 w=0 + dwmlp_c23_s0[1] AUC 0.79239 corr(z) 0.983 best blend +0.00000 w=0 + dwmlp_c23_s0[2] AUC 0.79391 corr(z) 0.992 best blend +0.00000 w=0 + dwmlp_c23_s0[3] AUC 0.79421 corr(z) 0.993 best blend +0.00000 w=0 + dwmlp_c23_s1[0] AUC 0.78901 corr(z) 0.971 best blend +0.00000 w=0 + dwmlp_c23_s1[1] AUC 0.79302 corr(z) 0.989 best blend +0.00000 w=0 + dwmlp_c23_s1[2] AUC 0.79371 corr(z) 0.991 best blend +0.00000 w=0 + dwmlp_c23_s1[3] AUC 0.79434 corr(z) 0.994 best blend +0.00000 w=0 + dwmlp_c23_s2[0] AUC 0.78762 corr(z) 0.967 best blend +0.00000 w=0 + dwmlp_c23_s2[1] AUC 0.79245 corr(z) 0.983 best blend +0.00000 w=0 + dwmlp_c23_s2[2] AUC 0.79413 corr(z) 0.992 best blend +0.00000 w=0 + dwmlp_c23_s2[3] AUC 0.79442 corr(z) 0.994 best blend +0.00000 w=0 + mlp2_c23_s0[0] AUC 0.78854 corr(z) 0.966 best blend +0.00000 w=0 + mlp2_c23_s0[1] AUC 0.79075 corr(z) 0.977 best blend +0.00000 w=0 + mlp2_c23_s0[2] AUC 0.79237 corr(z) 0.986 best blend +0.00000 w=0 + mlp2_c23_s0[3] AUC 0.79302 corr(z) 0.989 best blend +0.00000 w=0 + mlp2_c23_s1[0] AUC 0.78539 corr(z) 0.958 best blend +0.00000 w=0 + mlp2_c23_s1[1] AUC 0.79126 corr(z) 0.978 best blend +0.00000 w=0 + mlp2_c23_s1[2] AUC 0.79281 corr(z) 0.988 best blend +0.00000 w=0 + mlp2_c23_s1[3] AUC 0.79312 corr(z) 0.989 best blend +0.00000 w=0 + mlp2_c23_s2[0] AUC 0.78770 corr(z) 0.969 best blend +0.00000 w=0 + mlp2_c23_s2[1] AUC 0.79032 corr(z) 0.978 best blend +0.00000 w=0 + mlp2_c23_s2[2] AUC 0.79258 corr(z) 0.988 best blend +0.00000 w=0 + mlp2_c23_s2[3] AUC 0.79291 corr(z) 0.988 best blend +0.00000 w=0 + sC_s_c23_s0[0] AUC 0.79290 corr(z) 0.986 best blend +0.00000 w=0 + sC_s_c23_s0[1] AUC 0.79281 corr(z) 0.986 best blend +0.00000 w=0 + sC_s_c23_s1[0] AUC 0.79299 corr(z) 0.987 best blend +0.00000 w=0 + sC_s_c23_s1[1] AUC 0.79283 corr(z) 0.987 best blend +0.00000 w=0 + sCbig_c23_s0[0] AUC 0.79328 corr(z) 0.988 best blend +0.00000 w=0.05 + sCbig_c23_s0[1] AUC 0.79309 corr(z) 0.987 best blend +0.00000 w=0.05 + sCbig_c23_s0[2] AUC 0.79299 corr(z) 0.987 best blend +0.00001 w=0.05 + sCbig_c23_s1[0] AUC 0.79311 corr(z) 0.987 best blend +0.00000 w=0 + sCbig_c23_s1[1] AUC 0.79309 corr(z) 0.987 best blend +0.00000 w=0.05 + sCbig_c23_s1[2] AUC 0.79290 corr(z) 0.987 best blend +0.00000 w=0.05 diff --git a/dw_levers/logs/choice_base_25.log b/dw_levers/logs/choice_base_25.log new file mode 100644 index 0000000000000000000000000000000000000000..40964a9506915ceb672a9bb9172c3c53aec311fb --- /dev/null +++ b/dw_levers/logs/choice_base_25.log @@ -0,0 +1,11 @@ +v12 0.8172259475835104 +[ 7s] svd done +[ 12s] ep0 train 1.5714 hold 1.5278 8s valAUC 0.77560 +[ 17s] ep1 train 0.9808 hold 1.8055 12s valAUC 0.74639 +[ 22s] ep2 train 0.5487 hold 2.3603 17s valAUC 0.72596 +[ 26s] ep3 train 0.3437 hold 2.9096 22s valAUC 0.71568 +choice AUC 0.7756003662421029 within 0.7756003662421029 +blend z+w*lp 0.1 0.8172855210762608 +blend z+w*lp 0.2 0.8166980845190918 +blend z+w*lp 0.3 0.8157374588477733 +blend z+w*lp 0.5 0.8133236048636625 diff --git a/dw_levers/logs/stack25.log b/dw_levers/logs/stack25.log new file mode 100644 index 0000000000000000000000000000000000000000..43a98149db483f299229ee930cb1bfcd104d0642 --- /dev/null +++ b/dw_levers/logs/stack25.log @@ -0,0 +1,6 @@ +[ 6s] svd done +[ 12s] ep0 train 1.6461 hold 1.4576 8s valAUC 0.77770 +z not prob? -7.762106418609619 1.4581687450408936 +ctrl AUC 0.8173432554565818 delta 0.00011730787307140389 +stack AUC 0.817422278088598 delta 0.0001963305050876052 +stack-ctrl 8.023554142218892e-05 [2.16149252e-05 1.50093019e-04] diff --git a/dw_levers/logs/sweep1.log b/dw_levers/logs/sweep1.log new file mode 100644 index 0000000000000000000000000000000000000000..6bd21f204ce80c7bc99bd972c3839faf1f2a6fc3 --- /dev/null +++ b/dw_levers/logs/sweep1.log @@ -0,0 +1,36 @@ +CFG {"d": 32, "ep": 3, "lr": 0.001} +[ 12s] svd done +[ 18s] ep0 train 1.6461 hold 1.4576 13s valAUC 0.77770 +[ 22s] ep1 train 1.1656 hold 1.5018 18s valAUC 0.77658 +[ 27s] ep2 train 0.8589 hold 1.7046 23s valAUC 0.76555 +RES {"cfg": {"d": 32, "ep": 3, "lr": 0.001}, "cut": 25, "auc": 0.7776973973402646, "blend": {"0.1": -8e-05, "0.2": -0.00076, "0.4": -0.00295}} +CFG {"d": 32, "ep": 3, "lr": 0.001, "l2": 0.01} +[ 30s] svd done +[ 36s] ep0 train 1.6652 hold 1.4589 9s valAUC 0.77774 +[ 42s] ep1 train 1.1864 hold 1.5036 15s valAUC 0.77665 +[ 48s] ep2 train 0.8811 hold 1.7076 21s valAUC 0.76554 +RES {"cfg": {"d": 32, "ep": 3, "lr": 0.001, "l2": 0.01}, "cut": 25, "auc": 0.777737651041082, "blend": {"0.1": -8e-05, "0.2": -0.00075, "0.4": -0.00294}} +CFG {"d": 32, "ep": 3, "lr": 0.001, "l2": 0.1} +[ 51s] svd done +[ 57s] ep0 train 1.8282 hold 1.4700 8s valAUC 0.77784 +[ 63s] ep1 train 1.3527 hold 1.5188 14s valAUC 0.77686 +[ 69s] ep2 train 1.0524 hold 1.7287 20s valAUC 0.76530 +RES {"cfg": {"d": 32, "ep": 3, "lr": 0.001, "l2": 0.1}, "cut": 25, "auc": 0.7778433663501874, "blend": {"0.1": -7e-05, "0.2": -0.00071, "0.4": -0.00281}} +CFG {"d": 32, "ep": 3, "lr": 0.001, "hist_drop": 0.3, "l2": 0.01} +[ 72s] svd done +[ 78s] ep0 train 1.6956 hold 1.4836 9s valAUC 0.77525 +[ 84s] ep1 train 1.2712 hold 1.6013 15s valAUC 0.77353 +[ 90s] ep2 train 1.0243 hold 1.8377 21s valAUC 0.76473 +RES {"cfg": {"d": 32, "ep": 3, "lr": 0.001, "hist_drop": 0.3, "l2": 0.01}, "cut": 25, "auc": 0.775251609736144, "blend": {"0.1": -0.00018, "0.2": -0.00114, "0.4": -0.004}} +CFG {"d": 32, "ep": 3, "lr": 0.001, "att": true, "l2": 0.01} +[ 92s] svd done +[ 100s] ep0 train 1.8460 hold 1.7163 9s valAUC 0.76406 +[ 107s] ep1 train 1.5098 hold 1.5874 16s valAUC 0.76832 +[ 113s] ep2 train 1.2045 hold 1.5901 23s valAUC 0.76180 +RES {"cfg": {"d": 32, "ep": 3, "lr": 0.001, "att": true, "l2": 0.01}, "cut": 25, "auc": 0.768316048958352, "blend": {"0.1": -8e-05, "0.2": -0.00059, "0.4": -0.00232}} +CFG {"d": 64, "ep": 3, "lr": 0.001, "l2": 0.03} +[ 116s] svd done +[ 125s] ep0 train 1.6851 hold 1.4572 11s valAUC 0.77724 +[ 134s] ep1 train 1.1671 hold 1.5469 20s valAUC 0.77368 +[ 142s] ep2 train 0.6993 hold 1.9402 28s valAUC 0.75787 +RES {"cfg": {"d": 64, "ep": 3, "lr": 0.001, "l2": 0.03}, "cut": 25, "auc": 0.7772370992714306, "blend": {"0.1": -7e-05, "0.2": -0.00076, "0.4": -0.00298}} diff --git a/dw_levers/logs/sweep2.log b/dw_levers/logs/sweep2.log new file mode 100644 index 0000000000000000000000000000000000000000..52cc24b7149b11288fc9e1ff69a4e5ee91aa2440 --- /dev/null +++ b/dw_levers/logs/sweep2.log @@ -0,0 +1,27 @@ +CFG {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.0} +[ 9s] svd done +[ 15s] ep0 train 1.6042 hold 1.5460 9s valAUC 0.75228 +[ 19s] ep1 train 1.5421 hold 1.5458 14s valAUC 0.75218 +[ 24s] ep2 train 1.5410 hold 1.5446 19s valAUC 0.75248 +[ 28s] ep3 train 1.5401 hold 1.5441 23s valAUC 0.75263 +[ 33s] ep4 train 1.5393 hold 1.5436 28s valAUC 0.75278 +[ 37s] ep5 train 1.5387 hold 1.5473 32s valAUC 0.75320 +RES {"cfg": {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.0}, "cut": 25, "auc": 0.7527770901981062, "blend": {"0.1": -0.00053, "0.2": -0.00182, "0.4": -0.00551}} +CFG {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.05} +[ 40s] svd done +[ 45s] ep0 train 1.5878 hold 1.5150 7s valAUC 0.75848 +[ 49s] ep1 train 1.4516 hold 1.4935 11s valAUC 0.76314 +[ 54s] ep2 train 1.3794 hold 1.4803 16s valAUC 0.76700 +[ 58s] ep3 train 1.3101 hold 1.4729 20s valAUC 0.76961 +[ 62s] ep4 train 1.2438 hold 1.4816 25s valAUC 0.77129 +[ 67s] ep5 train 1.1798 hold 1.4903 29s valAUC 0.77254 +RES {"cfg": {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.05}, "cut": 25, "auc": 0.7696136210745811, "blend": {"0.1": -0.00039, "0.2": -0.00168, "0.4": -0.00529}} +CFG {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.15, "hist_drop": 0.2} +[ 70s] svd done +[ 74s] ep0 train 1.5853 hold 1.4910 7s valAUC 0.76688 +[ 79s] ep1 train 1.3515 hold 1.4941 11s valAUC 0.77235 +[ 83s] ep2 train 1.1926 hold 1.5369 16s valAUC 0.77374 +[ 87s] ep3 train 1.0468 hold 1.6133 20s valAUC 0.77222 +[ 92s] ep4 train 0.9117 hold 1.7607 25s valAUC 0.76832 +[ 97s] ep5 train 0.7875 hold 1.9241 29s valAUC 0.76389 +RES {"cfg": {"d": 32, "ep": 6, "lr": 0.003, "elr": 0.15, "hist_drop": 0.2}, "cut": 25, "auc": 0.766880051741821, "blend": {"0.1": -0.00038, "0.2": -0.00157, "0.4": -0.00497}} diff --git a/dw_levers/prep_data.py b/dw_levers/prep_data.py new file mode 100644 index 0000000000000000000000000000000000000000..60a5d0abced26b0632e788b4b2b3b191c99c1969 --- /dev/null +++ b/dw_levers/prep_data.py @@ -0,0 +1,9 @@ +import sys; sys.path.insert(0,'/workspace/huawei/dw_levers') +import numpy as np, time +from dwcommon import * +t=time.time(); D=Data(); print('load',time.time()-t, D.NE, D.NU, D.NI, D.Hlen.max(), D.Hlen.mean(), D.ETS.min()-J20, D.ETS.max()-J20, flush=True) +print(np.searchsorted(D.ETS,Jd(25)), np.searchsorted(D.ETS,Jd(23))) +v=D.val(25); print(len(v['y']), v['ts'].min()-Jd(25), v['ts'].max()-Jd(25)) +# is history ordered oldest first? +e=100000; print(D.HT[D.OFF[e]:D.OFF[e+1]]-D.ETS[e]) +import resource; print(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1e6,'GB') diff --git a/dw_levers/recal.py b/dw_levers/recal.py new file mode 100644 index 0000000000000000000000000000000000000000..3fbc47e4d6dc8cf1a2ae2ba0b1fa2903925185a7 --- /dev/null +++ b/dw_levers/recal.py @@ -0,0 +1,19 @@ +import sys, numpy as np, polars as pl, pandas as pd +sys.path.insert(0,'/workspace/huawei/dw_levers') +from common import z12, A +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +H='/workspace/huawei' +for c in (25,23): + m=pl.read_parquet(f'{H}/v10/feat_v3/val{c}.parquet',columns=['clicked','u','us_hlen','it_imp','us_nev','us_cold']).to_pandas() + z=z12(c); y=m.clicked.values.astype(int); base=A(y,z) + for name,keys in (('hlen',['us_hlen']),('imp',['it_imp']),('hlen x imp',['us_hlen','it_imp']),('nev x imp',['us_nev','it_imp'])): + cell=np.zeros(len(y),int) + for k in keys: cell=cell*8+pd.qcut(m[k],6,duplicates='drop',labels=False).values.astype(int) + out=np.zeros(len(y)) + for tr,te in GroupKFold(5).split(z,y,m.u.values): # honest CV by user + for cl in np.unique(cell): + a=tr[cell[tr]==cl]; b=te[cell[te]==cl] + if len(b)==0: continue + lr=LogisticRegression(C=10).fit(z[a,None],y[a]); out[b]=lr.decision_function(z[b,None]) + print(f'C{c} {name:12s} Δ={A(y,out)-base:+.5f}',flush=True) diff --git a/dw_levers/run_choice.py b/dw_levers/run_choice.py new file mode 100644 index 0000000000000000000000000000000000000000..b5443cf7ca559b9d2d4c5bdcb29f11db486108a8 --- /dev/null +++ b/dw_levers/run_choice.py @@ -0,0 +1,12 @@ +import sys, json, numpy as np +sys.path.insert(0,'/workspace/huawei/dw_levers') +from choice import * +from common import meta, z12, A +cut=int(sys.argv[1]); cfg=json.loads(sys.argv[2]); tag=sys.argv[3] +D=Data(); v=D.val(cut); m_=meta(cut); assert (m_['clicked'].to_numpy()==v['y']).all() +v['col']=align(D,v); z=z12(cut); print('v12',A(v['y'],z),flush=True) +mdl=fit_choice(D,Jd(cut),cfg,val=v) +lp,on=score(mdl,D,v['e'][::10]); lp=to_val(lp,v['col']); y=v['y'] +print('choice AUC',A(y,lp.ravel()),'within',A(y,lp.ravel()-np.repeat(0,len(y))),flush=True) +np.save(f'/workspace/huawei/dw_levers/cache/choice_{tag}_c{cut}.npy',np.column_stack([lp.ravel(),np.repeat(on,10)]).astype(np.float32)) +for w in (0.1,0.2,0.3,0.5): print('blend z+w*lp',w,A(y,z+w*lp.ravel()),flush=True) diff --git a/dw_levers/seg.py b/dw_levers/seg.py new file mode 100644 index 0000000000000000000000000000000000000000..4c157beec3fd3929ae8e6579dfd969d29c60dc7b --- /dev/null +++ b/dw_levers/seg.py @@ -0,0 +1,29 @@ +import sys, numpy as np, polars as pl, json +sys.path.insert(0,'/workspace/huawei/dw_levers') +from common import z12, A +H='/workspace/huawei' +out={} +for c in (25,23): + m=pl.read_parquet(f'{H}/v10/feat_v3/val{c}.parquet',columns=['rowid','eid','clicked','u','tm_stale','us_hlen','hist_n_win','it_imp','us_nev','tm_hour','it_ctr','hist_in','us_cold','us_nohist']).to_pandas() + z=z12(c); y=m.clicked.values.astype(int); base=A(y,z); print(f'== C{c} base {base:.5f} n={len(y)} pos={y.mean():.4f}') + # global AUC when segment scores replaced by oracle-shifted: seg pos get +big? use: pos in seg -> z+3 , neg in seg -> z-3 + def segs(col,bins): + q=pd_cut(m[col].values,bins); return q + import pandas as pd + pd_cut=lambda x,b: pd.cut(x,b,include_lowest=True).astype(str) + S={'tm_stale':pd_cut(m.tm_stale.values,[-1e9,0,6,12,24,48,1e9]) , + 'us_hlen':pd.qcut(m.us_hlen,6,duplicates='drop').astype(str), + 'hist_n_win':pd.qcut(m.hist_n_win,6,duplicates='drop').astype(str), + 'it_imp':pd.qcut(m.it_imp,6,duplicates='drop').astype(str), + 'us_nev':pd.qcut(m.us_nev,6,duplicates='drop').astype(str), + 'hour':pd_cut(m.tm_hour.values,[-1,5,11,17,24])} + for k,sg in S.items(): + print('--',k) + for s in sorted(set(sg)): + i=sg==s + if i.sum()<500 or y[i].min()==y[i].max(): continue + wi=A(y[i],z[i]); zz=z.copy(); zz[i&(y==1)]+=2; zz[i&(y==0)]-=2 # soft oracle + # recalibration headroom: shift segment so mean logit gap matches + hs=A(y,zz)-base + # mean score (pos rate) vs mean pred + print(f' {s:>28} share {i.mean():.3f} pos {y[i].mean():.3f} AUCin {wi:.4f} meanZ {z[i].mean():+.2f} oracleΔ {hs:+.4f} oracleΔ/share {hs/i.mean():+.4f}') diff --git a/dw_levers/stack_choice.py b/dw_levers/stack_choice.py new file mode 100644 index 0000000000000000000000000000000000000000..45ba9ac36d87b65c0f7ac92622a3fad320647f0c --- /dev/null +++ b/dw_levers/stack_choice.py @@ -0,0 +1,34 @@ +import sys, json, numpy as np, lightgbm as lgb +sys.path.insert(0,'/workspace/huawei/dw_levers') +from choice import * +from common import meta, z12, A +from sklearn.model_selection import GroupKFold +cut=int(sys.argv[1]); cfg=json.loads(sys.argv[2]); tag=sys.argv[3] +D=Data(); v=D.val(cut); v['col']=align(D,v); z=z12(cut); y=v['y'] +import os; f=f'cache/choice_{tag}_c{cut}.npz' +if os.path.exists(f): d=np.load(f); lp,on=d['lp'],d['on'] +else: + mdl=fit_choice(D,Jd(cut),cfg,val=v); lp,on=score(mdl,D,v['e'][::10]); lp=to_val(lp,v['col']).ravel(); on=on; np.savez(f,lp=lp,on=on) +n=len(y)//10; ev=np.repeat(np.arange(n),10) +lz=np.log(z/(1-z)) if z.max()<=1 else z +if z.max()>1: print('z not prob?',z.min(),z.max()) +def slate(a): m=a.reshape(n,10); return (m-m.mean(1,keepdims=True)).ravel(), (np.argsort(np.argsort(-m,1),1)).ravel() +Z=np.c_[lz,*slate(lz),np.repeat(np.log(np.exp(lz.reshape(n,10)).sum(1)),10)] +X1=np.c_[Z,lp,*slate(lp),np.repeat(on,10)] +users=v['user']; res={} +def cv(X,name): + out=np.zeros(len(y)) + for seed in (0,1): + for tr,te in GroupKFold(5).split(X,y,users): + p=dict(objective='binary',learning_rate=0.02,num_leaves=8,min_data_in_leaf=300,feature_fraction=0.8,bagging_fraction=0.8,bagging_freq=1,lambda_l2=10,seed=seed,verbose=-1,num_threads=4) + m=lgb.train(p,lgb.Dataset(X[tr],y[tr],init_score=lz[tr]),150) + out[te]+=m.predict(X[te],raw_score=True)+lz[te] + out/=2; res[name]=out; return out +base=A(y,lz) +for name,X in (('ctrl',Z),('stack',X1)): + o=cv(X,name); print(name,'AUC',A(y,o),'delta',A(y,o)-base,flush=True) +# paired event bootstrap of stack-ctrl +rng=np.random.default_rng(0); yy=y.reshape(n,10); a=res['stack'].reshape(n,10); b=res['ctrl'].reshape(n,10); ds=[] +for _ in range(100): + i=rng.integers(0,n,n); ds.append(A(yy[i].ravel(),a[i].ravel())-A(yy[i].ravel(),b[i].ravel())) +print('stack-ctrl',np.mean(ds),np.percentile(ds,[2.5,97.5])) diff --git a/dw_levers/sweep_choice.py b/dw_levers/sweep_choice.py new file mode 100644 index 0000000000000000000000000000000000000000..5c7faf5ce1007929ca410698cbb0e655e38f5f17 --- /dev/null +++ b/dw_levers/sweep_choice.py @@ -0,0 +1,11 @@ +import sys, json, numpy as np +sys.path.insert(0,'/workspace/huawei/dw_levers') +from choice import * +from common import meta, z12, A +cut=int(sys.argv[1]); cfgs=[json.loads(l) for l in open(sys.argv[2]) if l.strip()] +D=Data(); v=D.val(cut); v['col']=align(D,v); z=z12(cut) +for cfg in cfgs: + print('CFG',json.dumps(cfg),flush=True) + mdl=fit_choice(D,Jd(cut),cfg,val=v); lp,on=score(mdl,D,v['e'][::10]); lp=to_val(lp,v['col']).ravel(); y=v['y'] + r=dict(cfg=cfg,cut=cut,auc=A(y,lp),blend={w:round(A(y,z+w*lp)-A(y,z),5) for w in (0.1,0.2,0.4)}); print('RES',json.dumps(r),flush=True) + open('choice_sweep.jsonl','a').write(json.dumps(r)+'\n') diff --git a/dw_v12sig/findings.md b/dw_v12sig/findings.md new file mode 100644 index 0000000000000000000000000000000000000000..008757b22499c12729b83929481e8d89ace732bb --- /dev/null +++ b/dw_v12sig/findings.md @@ -0,0 +1,41 @@ +# v12 new-signal search (dw_v12sig) — findings + +Harness: dw_lb/lib.py residual LightGBM on v12 logits (init_score=z), user-grouped 5-fold, event-bootstrap 95% CI of AUC(test)-AUC(control). C25 primary, C23 confirm. v12 val AUC: C25 0.81723, C23 0.79566. +Noise control (null event-noise feature): C25 -0.00006 [-0.00012,-0.000005]; C23 -0.00002 [-0.00008,+0.00005]. Floor about ±0.0001. + +## Ranked results (delta AUC vs control, 95% CI) +| Group | C25 | C23 | +|---|---|---| +| tmp: per-dim taste products (13) | +0.00019 [+0.00002,+0.00033] | +0.00028 [+0.00012,+0.00044] | +| tmprod: products + item coords (25) | +0.00017 [0,+0.00034] | +0.00035 [+0.00014,+0.00051] | +| tm: all taste-metric (30) | +0.00016 [-0.00002,+0.00032] | +0.00036 [+0.00017,+0.00053] | +| all (68) | +0.00015 [-0.00004,+0.00030] | +0.00044 [+0.00020,+0.00064] | +| tmi: item coords only (12) | +0.00002 [-0.00015,+0.00016] | +0.00014 [-0.00003,+0.00030] | +| tmbil: learned bilinear (5) | +0.00004 [-0.00008,+0.00016] | not run | +| cl: conditional-logit item strength (15) | +0.00002 [-0.00013,+0.00016] | -0.00001 [-0.00014,+0.00012] | +| sq: transitions/Markov (12) | +0.00002 [-0.00012,+0.00017] | not run | +| sigs all (38) | +0.00001 [-0.00015,+0.00015] | not run | +| rp: repeat propensity (4) | -0.00002 [-0.00011,+0.00007] | not run | +| lc: item lifecycle (7) | -0.00009 [-0.00020,+0.00001] | not run | + +## Disagreement profile v12 vs p33 (no labels; t1_disagree.py, t1.log) +- Spearman 0.929, normal-score Pearson 0.935, residual sd 0.354; residual 15% event-level, 85% within-event. +- Spread across segments. Lowest agreement for no-history users (corr 0.47, tiny share). p33 ranks candidates already in the user's own history about +0.22 sd higher. +- Persistent ITEM-LEVEL residual: split-half reliability 0.36 (true item effect sd about 0.086), nearly uncorrelated (~0.07) with item CTR, popularity, lifecycle and CL strength. Hypothesis: p33 uses item identity / embedding (taste-region) information that v12 lacks. Matches the only winner: taste-dimension products from impression-space SVD. +- Event-level residual is 24% explainable from existing features (mostly SVD-family); within-event 7%. + +## Task 4 (no/short history) +Not run as an experiment: no-history rows are 1.3% of eval (6,650), so even a big lift there caps at about +0.0005 global AUC. +## Task 5 (slate/position) +No signal. Click rate by file position is flat (chi2 p=0.84), rows inside an event are not sorted, rowid is a hash. Nothing to exploit, nothing leaky to worry about. + +## Verdict +Nothing reaches +0.001. The only signal above noise is per-dimension user×item taste products (tm_prod0-11 + tm_prod_tail): about +0.0002 (C25, CI touches 0) and +0.0003 (C23, CI clear of 0), confirmed on both cuts, in the same direction. Item strength, lifecycle, transitions, repeat propensity, the learned bilinear metric and item-only coordinates are at the noise floor. Expect a small gain (~+0.0002–0.0004) if tm_prod is added as extra stacker features; probably worth it only if cheap. Likely v12 and p33 already capture the same sequence/item-strength information. + +## Leak safety / tests +Features use only train events before the cutoff plus each row's own history and slate; no eval aggregates (c27 files are built from train events only). test_sigs.py (run, passes): taste_metric recomputed for val25 matches cache within 1e-3; item_strength is unchanged when post-cutoff labels are flipped. + +## Files and reuse +- Code: sigs.py (extend_eval, item_strength, item_lifecycle, sequence, all_new), sigs2.py (imp_svd, learn_bilinear, taste_metric — `from sigs2 import taste_metric`; keep columns tm_prod*), run_groups.py (`run_groups.py `), test_sigs.py, t1_disagree.py, t1b_item.py, t1c.py, t5_slate.py. +- Features: cache/sig_c{23,25,27}.parquet, cache/tm_c{23,25,27}.parquet (c27 = eval-time). +- Results: results.jsonl, run*.log, t1.log, t1_resid.parquet. diff --git a/dw_v12sig/results.jsonl b/dw_v12sig/results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0fd3a09b4252397d27d67490d059b8a125e94572 --- /dev/null +++ b/dw_v12sig/results.jsonl @@ -0,0 +1,19 @@ +{"name": "v12sig_noise", "cut": 25, "nfeat": 1, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8171585501683595, "d_vs_z": -6.739741515093822e-05, "d_vs_ctrl": -6.094809697987369e-05, "ci_vs_ctrl": [-0.00012172757771286569, -5.225814527426415e-06], "sd": 3.1405712970425335e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6659201019047909} +{"name": "v12sig_cl", "cut": 25, "nfeat": 15, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172431409698145, "d_vs_z": 1.7193386304126257e-05, "d_vs_ctrl": 2.364270447519079e-05, "ci_vs_ctrl": [-0.0001276006095983051, 0.00015528683873076987], "sd": 7.111215168176308e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.666382687623516} +{"name": "v12sig_lc", "cut": 25, "nfeat": 7, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8171258842776561, "d_vs_z": -0.00010006330585432544, "d_vs_ctrl": -9.36139876832609e-05, "ci_vs_ctrl": [-0.00019997879057406519, 1.3580106612468018e-05], "sd": 5.731122009925467e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6658895215004376} +{"name": "v12sig_sq", "cut": 25, "nfeat": 12, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172391254915123, "d_vs_z": 1.3177908001926752e-05, "d_vs_ctrl": 1.9627226172991286e-05, "ci_vs_ctrl": [-0.00011759695631434197, 0.00016744051936322672], "sd": 7.023741015122335e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6665369601144708} +{"name": "v12sig_rp", "cut": 25, "nfeat": 4, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8171958254800887, "d_vs_z": -3.0122103421681423e-05, "d_vs_ctrl": -2.367278525061689e-05, "ci_vs_ctrl": [-0.00011231654110588352, 6.724128526499896e-05], "sd": 4.797619615335261e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6659012941650297} +{"name": "v12sig_all", "cut": 25, "nfeat": 38, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172252933920461, "d_vs_z": -6.541914643332092e-07, "d_vs_ctrl": 5.795126706731324e-06, "ci_vs_ctrl": [-0.00014515511066469922, 0.00015161341756678718], "sd": 8.016159318248819e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6662606668408032} +{"name": "v12sig_tmbil", "cut": 25, "nfeat": 5, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172640377790762, "d_vs_z": 3.809019556577642e-05, "d_vs_ctrl": 4.453951373684095e-05, "ci_vs_ctrl": [-7.667979150120518e-05, 0.00016468177852301818], "sd": 6.477103521899297e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6665646547341095} +{"name": "v12sig_tmprod", "cut": 25, "nfeat": 25, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173926891560154, "d_vs_z": 0.00016674157250495814, "d_vs_ctrl": 0.00017319089067602267, "ci_vs_ctrl": [1.6364068469722884e-08, 0.00034153239137667545], "sd": 8.146970827146051e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6658928329105425} +{"name": "v12sig_tm", "cut": 25, "nfeat": 30, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173788290683497, "d_vs_z": 0.00015288148483927344, "d_vs_ctrl": 0.00015933080301033797, "ci_vs_ctrl": [-1.6180528216988562e-05, 0.00031689746401823157], "sd": 8.419880596862764e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6659289553585908} +{"name": "v12sig_noise", "cut": 23, "nfeat": 1, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7955304044107099, "d_vs_z": -0.00012537297675574433, "d_vs_ctrl": -1.5034052699869349e-05, "ci_vs_ctrl": [-7.943090645900763e-05, 5.394747265244915e-05], "sd": 3.201110798736299e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6442237185499589} +{"name": "v12sig_tmprod", "cut": 23, "nfeat": 25, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7958909109102567, "d_vs_z": 0.00023513352279103117, "d_vs_ctrl": 0.00034547244684690614, "ci_vs_ctrl": [0.00014232738845744997, 0.0005114640319105219], "sd": 9.420359906713397e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6454062268527134} +{"name": "v12sig_tm", "cut": 23, "nfeat": 30, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7959079036807816, "d_vs_z": 0.00025212629331594627, "d_vs_ctrl": 0.00036246521737182125, "ci_vs_ctrl": [0.0001694616255996123, 0.0005299075006823445], "sd": 9.685927937433816e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6456233333774741} +{"name": "v12sig_cl", "cut": 23, "nfeat": 15, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7955343542107168, "d_vs_z": -0.00012142317674879166, "d_vs_ctrl": -1.1084252692916685e-05, "ci_vs_ctrl": [-0.00014228909437703495, 0.00012075060785508584], "sd": 7.365958161425481e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6443536362448076} +{"name": "v12sig_all", "cut": 23, "nfeat": 68, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7959834800495287, "d_vs_z": 0.0003277026620630563, "d_vs_ctrl": 0.0004380415861189313, "ci_vs_ctrl": [0.00020191954181220106, 0.0006377027300764782], "sd": 0.00010542090322451897, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6459186103143261} +{"name": "v12sig_tmp", "cut": 25, "nfeat": 13, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8174051491361859, "d_vs_z": 0.00017920155267547866, "d_vs_ctrl": 0.0001856508708465432, "ci_vs_ctrl": [2.3133276748360526e-05, 0.00033354929847700953], "sd": 7.819032854526502e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6658932830483899} +{"name": "v12sig_tmi", "cut": 25, "nfeat": 12, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172367796270116, "d_vs_z": 1.0832043501163291e-05, "d_vs_ctrl": 1.7281361672227824e-05, "ci_vs_ctrl": [-0.0001509946153145647, 0.00016059998032812444], "sd": 8.126794229520459e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6655751715230213} +{"name": "v12sig_all", "cut": 25, "nfeat": 68, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173653104192783, "d_vs_z": 0.00013936283576787556, "d_vs_ctrl": 0.0001458121539389401, "ci_vs_ctrl": [-4.121430504122336e-05, 0.00030456759421955973], "sd": 8.878763031890897e-05, "evAUC_base": 0.6680507246702352, "evAUC_test": 0.6660854161434371} +{"name": "v12sig_tmp", "cut": 23, "nfeat": 13, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7958279435599476, "d_vs_z": 0.00017216617248194677, "d_vs_ctrl": 0.00028250509653782174, "ci_vs_ctrl": [0.00012370744361178964, 0.00043968947250715433], "sd": 8.595574449986403e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6450338154376516} +{"name": "v12sig_tmi", "cut": 23, "nfeat": 12, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956900671534669, "d_vs_z": 3.4289766001283084e-05, "d_vs_ctrl": 0.00014462869005715806, "ci_vs_ctrl": [-3.351939774114408e-05, 0.00030396100199877875], "sd": 8.505831299199525e-05, "evAUC_base": 0.644454956712242, "evAUC_test": 0.6449723392363685} diff --git a/dw_v12sig/run23.log b/dw_v12sig/run23.log new file mode 100644 index 0000000000000000000000000000000000000000..bbc047aaa1cb90a4a3f15808bf6732503050f39a --- /dev/null +++ b/dw_v12sig/run23.log @@ -0,0 +1,5 @@ +{"name": "v12sig_noise", "cut": 23, "nfeat": 1, "base": 0.79566, "control": 0.79555, "test": 0.79553, "d_vs_z": -0.00013, "d_vs_ctrl": -2e-05, "ci_vs_ctrl": [-7.943090645900763e-05, 5.394747265244915e-05], "sd": 3e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64422} +{"name": "v12sig_tmprod", "cut": 23, "nfeat": 25, "base": 0.79566, "control": 0.79555, "test": 0.79589, "d_vs_z": 0.00024, "d_vs_ctrl": 0.00035, "ci_vs_ctrl": [0.00014232738845744997, 0.0005114640319105219], "sd": 9e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64541} +{"name": "v12sig_tm", "cut": 23, "nfeat": 30, "base": 0.79566, "control": 0.79555, "test": 0.79591, "d_vs_z": 0.00025, "d_vs_ctrl": 0.00036, "ci_vs_ctrl": [0.0001694616255996123, 0.0005299075006823445], "sd": 0.0001, "evAUC_base": 0.64445, "evAUC_test": 0.64562} +{"name": "v12sig_cl", "cut": 23, "nfeat": 15, "base": 0.79566, "control": 0.79555, "test": 0.79553, "d_vs_z": -0.00012, "d_vs_ctrl": -1e-05, "ci_vs_ctrl": [-0.00014228909437703495, 0.00012075060785508584], "sd": 7e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64435} +{"name": "v12sig_all", "cut": 23, "nfeat": 68, "base": 0.79566, "control": 0.79555, "test": 0.79598, "d_vs_z": 0.00033, "d_vs_ctrl": 0.00044, "ci_vs_ctrl": [0.00020191954181220106, 0.0006377027300764782], "sd": 0.00011, "evAUC_base": 0.64445, "evAUC_test": 0.64592} diff --git a/dw_v12sig/run23c.log b/dw_v12sig/run23c.log new file mode 100644 index 0000000000000000000000000000000000000000..20f8604fd77e7da6bc5c680a6d5f45bb83d1052b --- /dev/null +++ b/dw_v12sig/run23c.log @@ -0,0 +1,2 @@ +{"name": "v12sig_tmp", "cut": 23, "nfeat": 13, "base": 0.79566, "control": 0.79555, "test": 0.79583, "d_vs_z": 0.00017, "d_vs_ctrl": 0.00028, "ci_vs_ctrl": [0.00012370744361178964, 0.00043968947250715433], "sd": 9e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64503} +{"name": "v12sig_tmi", "cut": 23, "nfeat": 12, "base": 0.79566, "control": 0.79555, "test": 0.79569, "d_vs_z": 3e-05, "d_vs_ctrl": 0.00014, "ci_vs_ctrl": [-3.351939774114408e-05, 0.00030396100199877875], "sd": 9e-05, "evAUC_base": 0.64445, "evAUC_test": 0.64497} diff --git a/dw_v12sig/run25.log b/dw_v12sig/run25.log new file mode 100644 index 0000000000000000000000000000000000000000..2cd56490b4a01a505213013c071a58982fc27990 --- /dev/null +++ b/dw_v12sig/run25.log @@ -0,0 +1,6 @@ +{"name": "v12sig_noise", "cut": 25, "nfeat": 1, "base": 0.81723, "control": 0.81722, "test": 0.81716, "d_vs_z": -7e-05, "d_vs_ctrl": -6e-05, "ci_vs_ctrl": [-0.00012172757771286569, -5.225814527426415e-06], "sd": 3e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66592} +{"name": "v12sig_cl", "cut": 25, "nfeat": 15, "base": 0.81723, "control": 0.81722, "test": 0.81724, "d_vs_z": 2e-05, "d_vs_ctrl": 2e-05, "ci_vs_ctrl": [-0.0001276006095983051, 0.00015528683873076987], "sd": 7e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66638} +{"name": "v12sig_lc", "cut": 25, "nfeat": 7, "base": 0.81723, "control": 0.81722, "test": 0.81713, "d_vs_z": -0.0001, "d_vs_ctrl": -9e-05, "ci_vs_ctrl": [-0.00019997879057406519, 1.3580106612468018e-05], "sd": 6e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66589} +{"name": "v12sig_sq", "cut": 25, "nfeat": 12, "base": 0.81723, "control": 0.81722, "test": 0.81724, "d_vs_z": 1e-05, "d_vs_ctrl": 2e-05, "ci_vs_ctrl": [-0.00011759695631434197, 0.00016744051936322672], "sd": 7e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66654} +{"name": "v12sig_rp", "cut": 25, "nfeat": 4, "base": 0.81723, "control": 0.81722, "test": 0.8172, "d_vs_z": -3e-05, "d_vs_ctrl": -2e-05, "ci_vs_ctrl": [-0.00011231654110588352, 6.724128526499896e-05], "sd": 5e-05, "evAUC_base": 0.66805, "evAUC_test": 0.6659} +{"name": "v12sig_all", "cut": 25, "nfeat": 38, "base": 0.81723, "control": 0.81722, "test": 0.81723, "d_vs_z": -0.0, "d_vs_ctrl": 1e-05, "ci_vs_ctrl": [-0.00014515511066469922, 0.00015161341756678718], "sd": 8e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66626} diff --git a/dw_v12sig/run25b.log b/dw_v12sig/run25b.log new file mode 100644 index 0000000000000000000000000000000000000000..ebdd86bfe39c6a3e5d92846da64c9ca5490afbed --- /dev/null +++ b/dw_v12sig/run25b.log @@ -0,0 +1,3 @@ +{"name": "v12sig_tmbil", "cut": 25, "nfeat": 5, "base": 0.81723, "control": 0.81722, "test": 0.81726, "d_vs_z": 4e-05, "d_vs_ctrl": 4e-05, "ci_vs_ctrl": [-7.667979150120518e-05, 0.00016468177852301818], "sd": 6e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66656} +{"name": "v12sig_tmprod", "cut": 25, "nfeat": 25, "base": 0.81723, "control": 0.81722, "test": 0.81739, "d_vs_z": 0.00017, "d_vs_ctrl": 0.00017, "ci_vs_ctrl": [1.6364068469722884e-08, 0.00034153239137667545], "sd": 8e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66589} +{"name": "v12sig_tm", "cut": 25, "nfeat": 30, "base": 0.81723, "control": 0.81722, "test": 0.81738, "d_vs_z": 0.00015, "d_vs_ctrl": 0.00016, "ci_vs_ctrl": [-1.6180528216988562e-05, 0.00031689746401823157], "sd": 8e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66593} diff --git a/dw_v12sig/run25c.log b/dw_v12sig/run25c.log new file mode 100644 index 0000000000000000000000000000000000000000..39ba331c286aa567a3b06436e83594adc4d7c34c --- /dev/null +++ b/dw_v12sig/run25c.log @@ -0,0 +1,3 @@ +{"name": "v12sig_tmp", "cut": 25, "nfeat": 13, "base": 0.81723, "control": 0.81722, "test": 0.81741, "d_vs_z": 0.00018, "d_vs_ctrl": 0.00019, "ci_vs_ctrl": [2.3133276748360526e-05, 0.00033354929847700953], "sd": 8e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66589} +{"name": "v12sig_tmi", "cut": 25, "nfeat": 12, "base": 0.81723, "control": 0.81722, "test": 0.81724, "d_vs_z": 1e-05, "d_vs_ctrl": 2e-05, "ci_vs_ctrl": [-0.0001509946153145647, 0.00016059998032812444], "sd": 8e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66558} +{"name": "v12sig_all", "cut": 25, "nfeat": 68, "base": 0.81723, "control": 0.81722, "test": 0.81737, "d_vs_z": 0.00014, "d_vs_ctrl": 0.00015, "ci_vs_ctrl": [-4.121430504122336e-05, 0.00030456759421955973], "sd": 9e-05, "evAUC_base": 0.66805, "evAUC_test": 0.66609} diff --git a/dw_v12sig/run_groups.py b/dw_v12sig/run_groups.py new file mode 100644 index 0000000000000000000000000000000000000000..3fa4e164cdb32a1008126ed501d997810e10f300 --- /dev/null +++ b/dw_v12sig/run_groups.py @@ -0,0 +1,12 @@ +import sys; sys.path.insert(0,'/workspace/huawei/dw_lb'); sys.dont_write_bytecode=True +from lib import * +import lib +cuts=[int(x) for x in sys.argv[1].split(',')]; groups=sys.argv[2].split(',') +for c in cuts: + F=load_cut(c); S=pl.read_parquet(f'/workspace/huawei/dw_v12sig/cache/sig_c{c}.parquet').join(pl.read_parquet(f'/workspace/huawei/dw_v12sig/cache/tm_c{c}.parquet'),on='rowid') if os.path.exists(f'/workspace/huawei/dw_v12sig/cache/tm_c{c}.parquet') else pl.read_parquet(f'/workspace/huawei/dw_v12sig/cache/sig_c{c}.parquet'); assert (S['rowid'].to_numpy()==F['rowid']).all() + cols=[x for x in S.columns if x!='rowid'] + G={'noise':None,'cl':[x for x in cols if x.startswith('cl_')],'lc':[x for x in cols if x.startswith('lc_')],'sq':[x for x in cols if x.startswith('sq_')],'rp':[x for x in cols if x.startswith('rp_')],'all':cols,'tm':[x for x in cols if x.startswith('tm_')],'tmp':[x for x in cols if x.startswith('tm_prod')],'tmi':[x for x in cols if x.startswith('tm_item')],'tmbil':[x for x in cols if x.startswith('tm_bil') or x=='tm_cos'],'tmprod':[x for x in cols if x.startswith('tm_prod') or x.startswith('tm_item')]} + for g in groups: + if g=='noise': X=np.repeat(np.random.RandomState(c).randn(len(F['y'])//10,1).astype(np.float32),10,1).reshape(-1,1) + else: X=S.select(G[g]).to_numpy().astype(np.float32) + lib.report(f'v12sig_{g}',F,X,log='../dw_v12sig/results.jsonl') diff --git a/dw_v12sig/sig23.log b/dw_v12sig/sig23.log new file mode 100644 index 0000000000000000000000000000000000000000..a1860d445aa40156c1f46882b824474763c7156f --- /dev/null +++ b/dw_v12sig/sig23.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +23 (501550, 39) 8 s diff --git a/dw_v12sig/sig25.log b/dw_v12sig/sig25.log new file mode 100644 index 0000000000000000000000000000000000000000..6d2dc827a8398f468e41ad88f7b692f58fe03ccd --- /dev/null +++ b/dw_v12sig/sig25.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +25 (439640, 39) 13 s diff --git a/dw_v12sig/sig27.log b/dw_v12sig/sig27.log new file mode 100644 index 0000000000000000000000000000000000000000..fb51e017c713d36a0808773862fcc14c1dc0118b --- /dev/null +++ b/dw_v12sig/sig27.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +27 (529390, 39) 17 s diff --git a/dw_v12sig/sigs.py b/dw_v12sig/sigs.py new file mode 100644 index 0000000000000000000000000000000000000000..4be29f2aca45413c2354d67f172940ae6cc231c3 --- /dev/null +++ b/dw_v12sig/sigs.py @@ -0,0 +1,124 @@ +"""dw_v12sig reusable, leak-safe feature functions. Signature: f(D, X, rows) -> {name: float32 array len(rows['item'])} +D: dw_signals/common.Data (train events; for eval cuts D is extended with eval events via extend_eval ONLY so own-history lookups work), +X: fit cutoff (unix s). Every aggregate uses train events with ts < X only (+ the click log rebuilt from them). Per-row inputs are the row's own +user, candidate, ts, own pre-ts history and own slate. No eval-row labels/impressions/histories are aggregated.""" +import sys, os; sys.dont_write_bytecode=True +sys.path.insert(0,'/workspace/huawei/dw_signals') +import numpy as np, polars as pl +from common import * + +def extend_eval(D): + """copy of v12/mkdw.py:extend_eval (append eval events so own-history lookups work). Returns rows dict.""" + ee=pl.read_parquet('/workspace/huawei/lite/evaluation_events.parquet').sort('ts','eid'); er=pl.read_parquet('/workspace/huawei/lite/evaluation_rows.parquet') + assert ee['ts'].min()>=D.ETS.max() + n=ee.height; eid2e={x:i for i,x in enumerate(ee['eid'].to_list())} + er=er.with_columns(pl.col('eid').replace_strict(eid2e).alias('_e')).sort('_e','rowid'); assert (np.bincount(er['_e'].to_numpy())==10).all() + pos=np.minimum(np.searchsorted(D.uu,ee['user'].to_numpy()),D.NU-1); seen=D.uu[pos]==ee['user'].to_numpy() + eu=np.where(seen,pos,D.NU).astype(np.int32) + ri=np.searchsorted(D.items,er['item'].to_numpy()).astype(np.int32) + hi=ee['h_items'].explode().drop_nulls().to_numpy(); ht=ee['h_ts'].explode().drop_nulls().to_numpy().astype(np.int64); hl=ee['h_items'].list.len().to_numpy() + hp=np.minimum(np.searchsorted(D.items,hi),D.NI-1); ok=D.items[hp]==hi + hl=hl.astype(np.int64); cs=np.concatenate([[0],np.cumsum(ok.astype(np.int64))]); ce=np.concatenate([[0],np.cumsum(hl)]); hl2=cs[ce[1:]]-cs[ce[:-1]] + D0=D.NE; D.ETS=np.concatenate([D.ETS,ee['ts'].to_numpy()]); D.EU=np.concatenate([D.EU,eu]); D.Hlen=np.concatenate([D.Hlen,hl2.astype(np.int64)]) + base=D.OFF[-1]; D.OFF=np.concatenate([D.OFF[:-1],base+np.concatenate([[0],np.cumsum(hl2)])]).astype(np.int64) + D.HI=np.concatenate([D.HI,hp[ok].astype(np.int32)]); D.HT=np.concatenate([D.HT,ht[ok]]) + D.RI=np.concatenate([D.RI,ri.reshape(n,10)]); D.Y=np.concatenate([D.Y,np.zeros((n,10),np.int8)]); D.NE=D0+n; D.NU=D.NU+1 + e=np.repeat(np.arange(D0,D0+n),10) + return dict(rowid=er['rowid'].to_numpy(),e=e,item=ri,user=D.EU[e],ts=D.ETS[e]) + +def _zwithin(x): + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + return ((m-mu)/sd).ravel().astype(np.float32),(m-mu).ravel().astype(np.float32) + +# ---------- 1. item choice-model strength (conditional logit with outside option, MM fit) ---------- +def fit_item_strength(D,X,lo=0.0,hi_iter=40,a=3.0): + """P(click i | slate)=g_i/(1+sum g_j); P(no click)=1/(1+sum g). MM updates g_i=(c_i+a*g0)/(sum_{e∋i}1/D_e + a). events with lo<=ts0 else 0 + S=D.RI[s:nX]; Y=D.Y[s:nX]; ci=S[Y.astype(bool)] + c=np.bincount(ci,minlength=D.NI).astype(np.float64); g=np.full(D.NI,0.07); g0=0.07 + for _ in range(hi_iter): + De=1+g[S].sum(1); w=np.bincount(S.ravel(),weights=np.repeat(1/De,10),minlength=D.NI) + g=(c+a*g0)/(w+a); g0=g.mean() if False else float(np.median(g[c>0])) if (c>0).any() else g0 + ni=np.bincount(S.ravel(),minlength=D.NI) + return g.astype(np.float32),ni + +def item_strength(D,X,rows): + out={}; S=D.RI[rows['e'][::10]] + for tag,lo,a in (('all',0.0,3.0),('all_s20',0.0,20.0),('l48',X-2*DAY,5.0)): + g,ni=fit_item_strength(D,X,lo,a=a); lg=np.log(g) + r=lg[rows['item']]; z,dm=_zwithin(r); out[f'cl_{tag}_lg']=r.astype(np.float32); out[f'cl_{tag}_z']=z; out[f'cl_{tag}_dm']=dm + De=np.repeat(np.log(1+g[S].sum(1)),10); out[f'cl_{tag}_logD']=De.astype(np.float32) # model-implied event any-click level (item-strength only) + out[f'cl_{tag}_mx']=np.repeat(lg[S].max(1),10).astype(np.float32) + return out + +# ---------- 2. item lifecycle ---------- +def item_lifecycle(D,X,rows): + nX=int(np.searchsorted(D.ETS,X)); S=D.RI[:nX]; ts=D.ETS[:nX] + first=np.full(D.NI,np.inf); last=np.full(D.NI,-np.inf) + tsr=np.repeat(ts,10); np.minimum.at(first,S.ravel(),tsr); np.maximum.at(last,S.ravel(),tsr) + # also first click in history (warm-up) is NOT impression info; use only impressions + it=rows['item']; t=rows['ts'].astype(np.float64); out={} + age=(t-first[it])/3600; age=np.where(np.isfinite(age),age,-1.0); out['lc_age_h']=age.astype(np.float32); out['lc_logage']=np.log1p(np.maximum(age,0)).astype(np.float32) + out['lc_new']=(~np.isfinite(first[it])).astype(np.float32) + gap=(t-last[it])/3600; out['lc_gap_last_imp_h']=np.where(np.isfinite(gap),gap,np.nan).astype(np.float32) + # CTR lift as a function of item age (global curve from train events =0,curve[rb],np.nan).astype(np.float32) + # item impression volume in the last 6/24h before X relative to its lifetime rate + r24=np.bincount(S[ts>=X-DAY].ravel(),minlength=D.NI); r6=np.bincount(S[ts>=X-6*3600].ravel(),minlength=D.NI); tot=np.bincount(S.ravel(),minlength=D.NI) + life=np.maximum((X-first)/3600,1.0); out['lc_rel24']=np.where(np.isfinite(first[it]),(r24[it]/24)/(tot[it]/life[it]+1e-3),np.nan).astype(np.float32) + out['lc_rel6']=np.where(np.isfinite(first[it]),(r6[it]/6)/(tot[it]/life[it]+1e-3),np.nan).astype(np.float32) + return out + +# ---------- 3. sequence: next-click transitions + per-item repeat propensity ---------- +def _pair_counts(D,snap,lag): + u,i,t=snap['cu'],snap['ci'],snap['ct']; o=np.lexsort((t,u)); u=u[o]; i=i[o] + m=u[lag:]==u[:-lag]; a=i[:-lag][m].astype(np.int64); b=i[lag:][m].astype(np.int64) + keys,cnt=np.unique(a*D.NI+b,return_counts=True); return keys,cnt +def _lookup(keys,cnt,q): + p=np.minimum(np.searchsorted(keys,q),len(keys)-1); return np.where(keys[p]==q,cnt[p],0).astype(np.float32) +def sequence(D,X,rows): + snap=D.snap(X); out={}; e=rows['e']; it=rows['item'].astype(np.int64); n=len(it) + nout=np.bincount(snap['ci'],minlength=D.NI).astype(np.float32); tot=float(len(snap['ci'])) + hl=D.Hlen[e]; end=D.OFF[e+1] + for lag in (1,2,3): + keys,cnt=_pair_counts(D,snap,lag) + src_n=np.bincount((keys//D.NI).astype(np.int64),weights=cnt,minlength=D.NI).astype(np.float32) # transitions out of item + for k in range(1,lag+1) if lag==1 else (lag,): + # k-th last history item -> candidate (lag-step transition) + h=np.where(hl>=k,D.HI[np.maximum(end-k,0)],-1).astype(np.int64); ok=h>=0 + c=_lookup(keys,cnt,np.where(ok,h*D.NI+it,0)); nh=np.where(ok,src_n[np.maximum(h,0)],0) + pmi=np.log((c+0.5)/(nh+5)*tot/(nout[it]+5)) + out[f'sq_l{lag}_k{k}_cnt']=np.where(ok,c,np.nan).astype(np.float32); out[f'sq_l{lag}_k{k}_pmi']=np.where(ok,pmi,np.nan).astype(np.float32) + out[f'sq_l{lag}_k{k}_pmi_z']=_zwithin(out[f'sq_l{lag}_k{k}_pmi'])[0] + # reverse direction: candidate -> h (item preceded by candidate => candidate is an 'antecedent' of last click) + cr=_lookup(keys,cnt,np.where(ok,it*D.NI+h,0)); out[f'sq_l{lag}_k{k}_rev']=np.where(ok,cr,np.nan).astype(np.float32) + # max transition pmi from any of last 5 history items (lag-1 sense: unordered co-click within 5 steps of the user's own log) + # per-item repeat propensity + u,i,t=snap['cu'],snap['ci'],snap['ct']; o=np.lexsort((t,i,u)); u=u[o]; i=i[o]; same=np.r_[False,(u[1:]==u[:-1])&(i[1:]==i[:-1])] + rep=np.bincount(i[same],minlength=D.NI).astype(np.float32); allc=np.bincount(i,minlength=D.NI).astype(np.float32) + rr=(rep+0.5)/(allc+8); out['rp_item_rate']=rr[it].astype(np.float32) + # own-history repeat context + A=D.HI; cnt_own=np.zeros(n,np.float32); lastgap=np.full(n,np.nan,np.float32) + ev=e[::10]; + for k,ee in enumerate(ev): + a,b=D.OFF[ee],D.OFF[ee+1] + if b>a: + hi=A[a:b]; slate=rows['item'][k*10:(k+1)*10] + for j,c in enumerate(slate): + m=hi==c + if m.any(): cnt_own[k*10+j]=m.sum(); lastgap[k*10+j]=(rows['ts'][k*10]-D.HT[a:b][m][-1])/3600 + out['rp_own_cnt']=cnt_own; out['rp_own_gap_h']=lastgap; out['rp_x_rate']=(cnt_own>0)*out['rp_item_rate'] + return out + +def all_new(D,X,rows): + F={}; F.update(item_strength(D,X,rows)); F.update(item_lifecycle(D,X,rows)); F.update(sequence(D,X,rows)) + return pl.DataFrame({k:np.asarray(v,np.float32) for k,v in F.items()}) + +if __name__=='__main__': + import time; c=int(sys.argv[1]); D=Data(); t=time.time() + if c==27: + nt=D.NE; V=extend_eval(D); assert int(np.searchsorted(D.ETS,Jd(27)))==nt + else: V=D.val(c) + df=all_new(D,Jd(c),V); df=df.with_columns(pl.Series('rowid',V['rowid'])); df.write_parquet(f'/workspace/huawei/dw_v12sig/cache/sig_c{c}.parquet'); print(c,df.shape,round(time.time()-t),'s',flush=True) diff --git a/dw_v12sig/sigs2.py b/dw_v12sig/sigs2.py new file mode 100644 index 0000000000000000000000000000000000000000..9b6e316c7deba7bc83f97d1ae244554868774667 --- /dev/null +++ b/dw_v12sig/sigs2.py @@ -0,0 +1,54 @@ +"""Taste-metric signals: raw per-dimension impression-SVD interactions + a learned bilinear metric (conditional logit on train window). +Leak-safe: impression SVD from events ts0] + k=Un.shape[1]; W=torch.nn.Parameter(torch.eye(k)*3.0); b0=torch.nn.Parameter(torch.zeros(1)); out0=torch.nn.Parameter(torch.zeros(1)) + Ut=torch.tensor(Un); Vt=torch.tensor(Vn); opt=torch.optim.Adam([W,b0,out0],lr=0.02) + for ep in range(epochs): + p=rs.permutation(len(ev)) + for i in range(0,len(p),8192): + e=ev[p[i:i+8192]]; u=Ut[D.EU[e]]; v=Vt[torch.tensor(D.RI[e].astype(np.int64))] # (B,k),(B,10,k) + sc=torch.einsum('bk,kl,bjl->bj',u,W,v); y=torch.tensor(D.Y[e].astype(np.float32)); clicked=y.sum(1) + logits=torch.cat([sc,(out0+0*b0).expand(len(e),1)],1); tgt=torch.where(clicked>0,y.argmax(1),torch.full_like(clicked,10,dtype=torch.long).long()) + loss=torch.nn.functional.cross_entropy(logits,tgt)+lam*((W-torch.eye(k)*3)**2).sum(); opt.zero_grad(); loss.backward(); opt.step() + return W.detach().numpy() + +def taste_metric(D,X,rows,k=24): + U,sv,V,nu=imp_svd(D,X,k); Un=norm(U*sv); Vn=norm(V*sv); u=rows['user']; it=rows['item']; out={} + has=(nu[np.minimum(u,len(nu)-1)]>0)&(ur',uu,W,vv); sc=np.where(has,sc,np.nan).astype(np.float32); out['tm_bil']=sc; z,dm=_zwithin(sc); out['tm_bil_z']=z; out['tm_bil_dm']=dm + out['tm_cos']=np.where(has,(uu*vv).sum(1),np.nan).astype(np.float32); out['tm_bil_minus_cos']=out['tm_bil']-out['tm_cos']*3 + return out + +if __name__=='__main__': + import time; c=int(sys.argv[1]); D=Data(); t=time.time() + if c==27: + nt=D.NE; V=extend_eval(D); assert int(np.searchsorted(D.ETS,Jd(27)))==nt + else: V=D.val(c) + df=pl.DataFrame({k:np.asarray(v,np.float32) for k,v in taste_metric(D,Jd(c),V).items()}).with_columns(pl.Series('rowid',V['rowid'])) + df.write_parquet(f'/workspace/huawei/dw_v12sig/cache/tm_c{c}.parquet'); print(c,df.shape,round(time.time()-t),'s',flush=True) diff --git a/dw_v12sig/t1.log b/dw_v12sig/t1.log new file mode 100644 index 0000000000000000000000000000000000000000..7815932cdcedc18d9867c8f2e89cb5774f0a07f1 --- /dev/null +++ b/dw_v12sig/t1.log @@ -0,0 +1,77 @@ +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_function_base_impl.py:3029: RuntimeWarning: Degrees of freedom <= 0 for slice + c = cov(x, y, rowvar, dtype=dtype) +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_function_base_impl.py:2902: RuntimeWarning: divide by zero encountered in divide + c *= np.true_divide(1, fact) +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_function_base_impl.py:2902: RuntimeWarning: invalid value encountered in multiply + c *= np.true_divide(1, fact) +spearman 0.9290753537605462 pearson normal scores 0.9353092488121516 +beta 0.9353110154870751 resid sd 0.35383088162634546 +var share event-level 0.14796606882180696 within 0.852033931178193 +## hist len +bin n corr mean|res| mean_res mean|within| mean|event| + 0 6650 0.474 0.142 +0.025 0.127 0.061 + 1 24890 0.897 0.307 -0.117 0.269 0.151 + 2 61790 0.916 0.268 -0.022 0.245 0.108 + 3 102880 0.930 0.269 +0.026 0.247 0.105 + 4 79200 0.937 0.277 +0.041 0.255 0.108 + 5 253980 0.942 0.290 -0.007 0.270 0.106 +## stale h from first-cutoff? +bin n corr mean|res| mean_res mean|within| mean|event| + 0 200 0.933 0.320 -0.019 0.302 0.095 + 5 250 0.945 0.293 +0.019 0.268 0.107 + 6 528940 0.935 0.280 -0.000 0.258 0.108 +## hours since last hist click +bin n corr mean|res| mean_res mean|within| mean|event| + 5 529390 0.935 0.280 -0.000 0.258 0.108 +## item impressions +bin n corr mean|res| mean_res mean|within| mean|event| + 1 5610 0.898 0.271 -0.044 0.251 0.108 + 2 123550 0.922 0.274 -0.036 0.255 0.106 + 3 210560 0.934 0.281 +0.007 0.258 0.107 + 4 128116 0.942 0.284 +0.025 0.261 0.109 + 5 57911 0.947 0.283 +0.002 0.260 0.111 + 6 3643 0.948 0.292 -0.040 0.265 0.110 +## item ctr +bin n corr mean|res| mean_res mean|within| mean|event| + 0 24316 0.911 0.275 -0.047 0.255 0.107 + 1 115632 0.926 0.280 -0.012 0.258 0.107 + 2 202972 0.936 0.281 +0.005 0.258 0.108 + 3 163519 0.941 0.281 +0.010 0.259 0.109 + 4 22793 0.945 0.279 +0.001 0.257 0.109 + 5 158 0.935 0.315 -0.042 0.288 0.105 +## ord (slate pos?) +bin n corr mean|res| mean_res mean|within| mean|event| + 0 1 nan 0.244 +0.244 0.195 0.048 + 1 2 1.000 0.323 -0.010 0.323 0.048 + 2 2 1.000 0.178 -0.019 0.178 0.048 + 3 2 1.000 0.201 -0.201 0.249 0.048 + 4 2 1.000 0.347 +0.347 0.298 0.048 + 5 529381 0.935 0.280 -0.000 0.258 0.108 +## user n events +bin n corr mean|res| mean_res mean|within| mean|event| + 1 14530 0.897 0.259 -0.087 0.225 0.130 + 2 66430 0.910 0.271 -0.031 0.246 0.113 + 3 111600 0.927 0.271 +0.022 0.249 0.106 + 4 108690 0.937 0.278 +0.032 0.256 0.107 + 5 228140 0.943 0.290 -0.012 0.269 0.106 +## cand in own history +bin n corr mean|res| mean_res mean|within| mean|event| + 0 515674 0.930 0.279 -0.006 0.257 0.108 + 1 13716 0.877 0.317 +0.222 0.310 0.107 +## item warm clicks +bin n corr mean|res| mean_res mean|within| mean|event| + 1 1520 0.872 0.271 -0.037 0.256 0.106 + 2 53711 0.910 0.277 -0.041 0.257 0.107 + 3 304920 0.932 0.279 -0.001 0.258 0.107 + 4 166735 0.945 0.283 +0.016 0.260 0.110 + 5 2504 0.949 0.292 -0.037 0.269 0.108 +## item trend +bin n corr mean|res| mean_res mean|within| mean|event| + 3 35273 0.941 0.280 +0.008 0.258 0.109 + 4 494117 0.935 0.280 -0.001 0.258 0.108 +res R2 oof from 166 feats 0.09675264339098932 +[('sv_k64_ncos', 5071), ('ts', 3156), ('us_nclk_all', 3055), ('sv_k32_imax20', 2605), ('sv_k16_hcos', 2324), ('sv_k32_hcos', 2178), ('sv_k64_con1', 2079), ('us_ctr', 1984), ('hist_pos', 1977), ('tm_dt', 1951), ('tm_stale', 1937), ('sv_k32_hnorm', 1849), ('ev_sv_k32_hcos_z', 1641), ('ev_sv_k64_hcos_z', 1625), ('ev_sv_k64_con05_z', 1355)] +res_event R2 oof from 166 feats 0.23561281045276106 +res_within R2 oof from 166 feats 0.07202565588122822 +['item', 'w_0', 'w_1', 'n_0', 'n_1'] +items with >=5 rows both halves 15295 split-half corr of item mean within-residual 0.36094488917985074 diff --git a/dw_v12sig/t1_disagree.py b/dw_v12sig/t1_disagree.py new file mode 100644 index 0000000000000000000000000000000000000000..e3d907d6ef2fa062d7680e04b254b604ff9216fa --- /dev/null +++ b/dw_v12sig/t1_disagree.py @@ -0,0 +1,49 @@ +import sys; sys.dont_write_bytecode=True +import numpy as np, polars as pl, lightgbm as lgb +from scipy.stats import norm, rankdata, spearmanr +from sklearn.model_selection import GroupKFold +H='/workspace/huawei' +ev=pl.read_parquet(f'{H}/v10/feat_v3/eval.parquet') +v12=pl.read_csv(f'{H}/v12/out/submission_v12.csv').rename({'predicted_probability':'v12'}) +p33=pl.read_parquet(f'{H}/teammate/work33/p33_recovered_ranks.parquet') +d=ev.join(v12,on='rowid').join(p33,on='rowid'); assert d.height==ev.height +n=d.height +nz=lambda r: norm.ppf((rankdata(r)-0.5)/len(r)) +a=nz(d['v12'].to_numpy()); b=nz(d['p33_rank'].to_numpy()) +print('spearman',spearmanr(a,b)[0],'pearson normal scores',np.corrcoef(a,b)[0,1]) +# residual of b on a (linear) +beta=np.cov(a,b)[0,1]/a.var(); res=b-beta*a; print('beta',beta,'resid sd',res.std()) +R=res.reshape(-1,10); assert (d['eid'].to_numpy().reshape(-1,10)==d['eid'].to_numpy().reshape(-1,10)[:,:1]).all() +evm=R.mean(1,keepdims=True); within=(R-evm).ravel(); evl=np.repeat(evm,10,1).ravel() +print('var share event-level',evl.var()/res.var(),'within',within.var()/res.var()) +d=d.with_columns(pl.Series('res',res),pl.Series('res_ev',evl),pl.Series('res_w',within),pl.Series('za',a),pl.Series('zb',b)) +d.select(['rowid','res','res_ev','res_w','za','zb']).write_parquet('t1_resid.parquet') +# segments: abs residual and spearman by segment +def seg(col,bins,name): + x=d[col].to_numpy(); q=np.digitize(x,bins); out=[] + for k in np.unique(q): + m=q==k; out.append((k,m.sum(),np.corrcoef(a[m],b[m])[0,1],np.abs(res[m]).mean(),res[m].mean(),np.abs(within[m]).mean(),np.abs(evl[m]).mean())) + print('##',name); print('bin n corr mean|res| mean_res mean|within| mean|event|') + for o in out: print(' %d %7d %.3f %.3f %+.3f %.3f %.3f'%o) +d=d.with_columns(pl.col('hist_n_all').fill_null(-1)) +seg('us_hlen',[1,3,6,11,16,21,30],'hist len'); seg('tm_stale',[1,6,12,24,48,72],'stale h from first-cutoff?') +seg('hist_since_clk',[1,6,24,72,168],'hours since last hist click'); seg('it_imp',[20,50,100,200,400,1000],'item impressions') +seg('it_ctr',[0.03,0.05,0.07,0.1,0.15],'item ctr'); seg('ord',[1,3,5,7,9],'ord (slate pos?)'); seg('us_nev',[0,1,3,6,10],'user n events'); seg('hist_in',[0.5],'cand in own history') +seg('it_nclk_warm',[0,1,5,20,100],'item warm clicks'); seg('it_trend24',[-0.5,-0.1,0.1,0.5],'item trend') +# modelling: what features explain the residual? +feats=[c for c in ev.columns if c not in ('rowid','eid','ii','u')] +X=d.select(feats).to_numpy().astype(np.float32); user=d['u'].to_numpy() +for tgt,nm in ((res,'res'),(evl,'res_event'),(within,'res_within')): + p=np.zeros(n) + for tr,te in GroupKFold(5).split(X,tgt,user): + m=lgb.train(dict(objective='regression',learning_rate=0.05,num_leaves=31,min_data_in_leaf=200,verbose=-1,num_threads=3,feature_fraction=0.7),lgb.Dataset(X[tr],tgt[tr]),250); p[te]=m.predict(X[te]) + print(nm,'R2 oof from %d feats'%len(feats),1-((tgt-p)**2).mean()/tgt.var()) + if nm=='res': + imp=sorted(zip(m.feature_importance('gain'),feats),reverse=True)[:15]; print([ (f,int(g)) for g,f in imp]) +# item-level random effect: split events by parity, correlate item mean residual (within-event) +item=d['ii'].to_numpy(); evi=np.repeat(np.arange(n//10),10); par=(evi%2) +df=pl.DataFrame(dict(item=item,par=par,w=within,r=res)) +g=df.group_by(['item','par']).agg(pl.col('w').mean().alias('w'),pl.col('w').len().alias('n')).pivot(index='item',on='par',values=['w','n']).drop_nulls() +print(g.columns); c=g.columns +m=(g[c[3]]>=5)&(g[c[4]]>=5) if len(c)>4 else None +gg=g.filter(m); print('items with >=5 rows both halves',gg.height,'split-half corr of item mean within-residual',np.corrcoef(gg[c[1]],gg[c[2]])[0,1]) diff --git a/dw_v12sig/t1b_item.py b/dw_v12sig/t1b_item.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac40a3f7ad647c28382d2914b0fd60169b047a3 --- /dev/null +++ b/dw_v12sig/t1b_item.py @@ -0,0 +1,12 @@ +import numpy as np, polars as pl +H='/workspace/huawei' +ev=pl.read_parquet(f'{H}/v10/feat_v3/eval.parquet'); r=pl.read_parquet('t1_resid.parquet') +d=ev.join(r,on='rowid'); +g=d.group_by('ii').agg(pl.col('res_w').mean().alias('w'),pl.col('res').mean().alias('r'),pl.len().alias('n'),pl.col('za').mean().alias('za'),pl.col('zb').mean().alias('zb'), + *[pl.col(c).first().alias(c) for c in ['it_imp','it_clk','it_ctr','it_ctr24','it_ctr6','it_trend24','it_imp24','it_imp_rate','it_nclk_all','it_nusers','it_nclk_warm','it_clk_dec2d','it_imptrend24']]) +g=g.filter(pl.col('n')>=8); print(g.height) +for c in g.columns[4:]: + x=g[c].to_numpy().astype(float); m=np.isfinite(x) + print('%-16s corr(w)=%+.3f corr(r)=%+.3f corr(zb-za)=%+.3f'%(c,np.corrcoef(x[m],g['w'].to_numpy()[m])[0,1],np.corrcoef(x[m],g['r'].to_numpy()[m])[0,1],np.corrcoef(x[m],(g['zb']-g['za']).to_numpy()[m])[0,1])) +print('corr(n_eval_rows, w)',np.corrcoef(g['n'],g['w'])[0,1], 'corr log n',np.corrcoef(np.log(g['n']),g['w'])[0,1]) +print('sd of item mean w',g['w'].std(),' expected noise sd',(d['res_w'].std()/np.sqrt(g['n'])).mean()) diff --git a/dw_v12sig/t1c.py b/dw_v12sig/t1c.py new file mode 100644 index 0000000000000000000000000000000000000000..1f091f222d43923fde9a5ed56fc655d320734f5c --- /dev/null +++ b/dw_v12sig/t1c.py @@ -0,0 +1,12 @@ +import numpy as np, polars as pl +S=pl.read_parquet('cache/sig_c27.parquet'); r=pl.read_parquet('t1_resid.parquet') +H='/workspace/huawei'; ev=pl.read_parquet(f'{H}/v10/feat_v3/eval.parquet',columns=['rowid','ii']) +# sigs order = extend_eval order (sorted by ts,eid) -> join on rowid +d=S.join(r,on='rowid').join(ev,on='rowid'); print(d.height) +for c in ['cl_all_lg','cl_all_z','cl_all_dm','cl_all_s20_dm','cl_l48_dm','lc_agecurve','lc_logage','lc_rel24','sq_l1_k1_pmi_z','rp_item_rate']: + x=d[c].to_numpy(); m=np.isfinite(x); print('%-16s row-level corr(res_w)=%+.3f corr(res)=%+.3f corr(res_ev)=%+.3f'%(c,np.corrcoef(x[m],d['res_w'].to_numpy()[m])[0,1],np.corrcoef(x[m],d['res'].to_numpy()[m])[0,1],np.corrcoef(x[m],d['res_ev'].to_numpy()[m])[0,1])) +for c in ['cl_all_logD','cl_all_mx','cl_l48_logD']: + x=d[c].to_numpy(); print(c,'corr res_ev %+.3f'%np.corrcoef(x,d['res_ev'].to_numpy())[0,1]) +# item-level +g=d.group_by('ii').agg(pl.col('res_w').mean().alias('w'),pl.len().alias('n'),pl.col('cl_all_dm').mean().alias('cl'),pl.col('cl_all_lg').first().alias('lg')).filter(pl.col('n')>=8) +print('item-level corr(mean res_w, cl logg)',np.corrcoef(g['w'],g['lg'])[0,1],g.height) diff --git a/dw_v12sig/t5_slate.py b/dw_v12sig/t5_slate.py new file mode 100644 index 0000000000000000000000000000000000000000..d4bae187aa41cdceef87deeabb4eccc41a412613 --- /dev/null +++ b/dw_v12sig/t5_slate.py @@ -0,0 +1,21 @@ +import numpy as np, polars as pl +H='/workspace/huawei' +for nm in ['train','evaluation']: + d=pl.read_csv(f'{H}/data/{nm}.csv',columns=['rowid','event_id','item_id']+(['clicked'] if nm=='train' else [])) + n=d.height; ev=d['event_id'].to_numpy() + contiguous=(ev.reshape(-1,10)==ev.reshape(-1,10)[:,:1]).all(); print(nm,n,'events contiguous 10-blocks',contiguous) + pos=np.tile(np.arange(10),n//10) + if nm=='train': + y=d['clicked'].to_numpy(); print('click rate by file position',np.round([y[pos==k].mean() for k in range(10)],4)) + # chi2 test + from scipy.stats import chisquare + c=np.array([y[pos==k].sum() for k in range(10)]); print('chi2 p',chisquare(c).pvalue) + # time-split: is first-half vs second-half same + # rowid hash order within event? sorted by rowid? + rid=d['rowid'].to_numpy().reshape(-1,10); print('rowid sorted within event frac',(np.diff(rid.astype('U70').view('U70') if False else np.argsort(rid,1),axis=1)!=1).any(1).mean()) + it=d['item_id'].to_numpy().reshape(-1,10); print('item sorted within event frac',np.mean([ (it[i]==np.sort(it[i])).all() for i in range(20000)])) + # position by item popularity + pop=d.group_by('item_id').len().rename({'len':'pop'}); dd=d.with_columns(pl.Series('pos',pos)).join(pop,on='item_id') + print(dd.group_by('pos').agg(pl.col('pop').mean()).sort('pos')['pop'].to_numpy().round(1)) + else: + it=d['item_id'].to_numpy().reshape(-1,10); print('item sorted within event frac',np.mean([ (it[i]==np.sort(it[i])).all() for i in range(20000)])) diff --git a/dw_v12sig/test_sigs.py b/dw_v12sig/test_sigs.py new file mode 100644 index 0000000000000000000000000000000000000000..1d6fc6cd42dc054adca48bbb0a1d56af816ead22 --- /dev/null +++ b/dw_v12sig/test_sigs.py @@ -0,0 +1,13 @@ +"""Small leak-safety / reproducibility test: features for val25 recomputed vs cache; and truncating train events after X must not change features.""" +import sys; sys.dont_write_bytecode=True +sys.path.insert(0,'/workspace/huawei/dw_v12sig') +from sigs2 import * +D=Data(); V=D.val(25); X=Jd(25) +df=pl.DataFrame({k:np.asarray(v,np.float32) for k,v in taste_metric(D,X,V).items()}) +ref=pl.read_parquet('/workspace/huawei/dw_v12sig/cache/tm_c25.parquet') +bad=[(k,float(np.nanmax(np.abs(df[k].to_numpy()-ref[k].to_numpy())))) for k in df.columns if np.nanmax(np.abs(df[k].to_numpy()-ref[k].to_numpy()))>1e-3] +print('tm mismatches vs cache (torch/svd tolerance 1e-3):',bad) +# leak test: change labels of all events with ts>=X -> features must be identical +Y0=D.Y.copy(); nX=int(np.searchsorted(D.ETS,X)); D.Y[nX:]=1-D.Y[nX:] +f1=item_strength(D,X,V); D.Y=Y0 +f0=item_strength(D,X,V); assert all(np.array_equal(f0[k],f1[k],equal_nan=True) for k in f0); print('item_strength unchanged when post-cutoff labels flipped: OK') diff --git a/dw_v12sig/tm23.log b/dw_v12sig/tm23.log new file mode 100644 index 0000000000000000000000000000000000000000..7a0960b6d408a62598ad15b1adaecee40c6db098 --- /dev/null +++ b/dw_v12sig/tm23.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +23 (501550, 31) 12 s diff --git a/dw_v12sig/tm25.log b/dw_v12sig/tm25.log new file mode 100644 index 0000000000000000000000000000000000000000..37732ee033642da1ac0adc119c81020f3c85220c --- /dev/null +++ b/dw_v12sig/tm25.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +25 (439640, 31) 10 s diff --git a/dw_v12sig/tm27.log b/dw_v12sig/tm27.log new file mode 100644 index 0000000000000000000000000000000000000000..979843cfbfc569c331e01ac30080011da01151be --- /dev/null +++ b/dw_v12sig/tm27.log @@ -0,0 +1,5 @@ +/workspace/huawei/dw_v12sig/sigs.py:30: RuntimeWarning: Mean of empty slice + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 +/workspace/venv/lib/python3.13/site-packages/numpy/lib/_nanfunctions_impl.py:1997: RuntimeWarning: Degrees of freedom <= 0 for slice. + var = nanvar(a, axis=axis, dtype=dtype, out=out, ddof=ddof, +27 (529390, 31) 7 s diff --git a/probes/make_probe.py b/probes/make_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..28f2869fe713646cfb9de4515710a2a0dedc8fc1 --- /dev/null +++ b/probes/make_probe.py @@ -0,0 +1,18 @@ +import numpy as np, pandas as pd, zipfile, hashlib +from scipy.stats import rankdata, spearmanr +v=pd.read_csv('/workspace/huawei/v12/out/submission_v12.csv') +q=pd.read_parquet('/workspace/huawei/teammate/work33/p33_recovered_ranks.parquet') +ref=pd.read_csv('/workspace/huawei/v5/submission_v5.csv').rowid # same rowid set/order as v8 (v8 csv lost in restore) +assert v.rowid.is_unique and q.rowid.is_unique and set(v.rowid)==set(q.rowid)==set(ref) and len(v)==529390 +q=q.set_index('rowid').loc[v.rowid] +a=v.predicted_probability.values; b=q.p33_rank.values +n=len(a); ra=(rankdata(a,'average')-1)/(n-1); rb=(rankdata(b,'average')-1)/(n-1) +print('spearman v12 vs p33',spearmanr(a,b)[0],'pearson of ranks',np.corrcoef(ra,rb)[0,1]) +s=0.8*ra+0.2*rb; print('score range',s.min(),s.max(),'finite',np.isfinite(s).all(),'ties',n-len(np.unique(s))) +s=np.clip(s,0,1) +df=pd.DataFrame({'rowid':v.rowid,'predicted_probability':s}); df=df.set_index('rowid').loc[ref].reset_index() +assert list(df.rowid)==list(ref) and df.predicted_probability.between(0,1).all() +df.to_csv('/workspace/huawei/probes/probe_v12_p33_w80.csv',index=False,float_format='%.9g') +zf=zipfile.ZipFile('/workspace/huawei/probes/probe_v12_p33_w80.csv.zip','w',zipfile.ZIP_DEFLATED); zf.write('/workspace/huawei/probes/probe_v12_p33_w80.csv',arcname='probe_v12_p33_w80.csv'); zf.close() +print(hashlib.sha256(open('/workspace/huawei/probes/probe_v12_p33_w80.csv.zip','rb').read()).hexdigest()) +print('spearman probe vs v12',spearmanr(df.predicted_probability,v.set_index('rowid').predicted_probability.loc[df.rowid])[0]) diff --git a/tools/package_final.py b/tools/package_final.py index 9d5478af4c0332a73554c365eeb16c627a3899ca..9095351f5d161a883fbda1b0f77482b6ec534685 100644 --- a/tools/package_final.py +++ b/tools/package_final.py @@ -16,7 +16,7 @@ metrics.json: either {"cut":25,"auc":..,"rw":..} or a list / jsonl of such rows import argparse, hashlib, json, math, os, sys, zipfile import pandas as pd -REF = "/workspace/huawei/v2work/out/submission_v8.csv" # reference rowid set +REF = "/workspace/huawei/data/evaluation.csv" # reference rowid set (eval set; same set as the former v2work/out/submission_v8.csv, lost in the 2026-10-28 box restore) N_ROWS = 529390 def load_c25(path): diff --git a/v12/NOTES.md b/v12/NOTES.md index 6ebf987a40b5ef0977d80d38300ebd5aeea9b4de..0931756528c38b04b9b2032dec451f38715f4f98 100644 --- a/v12/NOTES.md +++ b/v12/NOTES.md @@ -29,3 +29,28 @@ b_full = dwbig:1 + dw_s:1 + 2*dwmlp:3 (seeds 0,1,2), same recipe as v11 accept rule (C25 up, C23 not down >.0002) holds for both seeds. Submission uses seed-0 LGB + 3 MLP seeds (not the 2-seed LGB average). runtimes (sum of wall): DW feature builds ~534s, dw_s 361s, dwbig 1425s, dwmlp 599s(+prep), seed-1 baselines 415s, resid oof ~2min. Whole v12 ~1h45 incl. waits. scratch cleaned (__pycache__); kept feat_dw, ens, logs. + +## PACKAGING TASK (v12 -> sol/solution_v12.py) 00:50 Dublin +- Plan: generator sol_work/make_sol12.py builds sol/solution_v12.py from v10 sol (SRC dict embedded) + seq11 (SQ_SEED/H/HID) x11 seeds (0-7 base; 8-10 SQ_H=4 SQ_HID=128) + avg_sq + DW (common/signals/reusable/mkdw, paths->W) + ens (dw_s, dwbig) + mlp11 x3 seeds + blend (dwbig+dw_s+2*mlp mean ep4)/4. +- Workdir for verification: /tmp-like clean dir with only train.csv/evaluation.csv symlinked; keep output stats in sol_work/verify_stats.json + +## PACKAGING progress (00:57 Dublin) +- Built generator sol_work/make_sol12.py (+head12.txt, tail12.txt, src12/ modules) -> sol/solution_v12.py (no /workspace paths; stages incl. feat_sq (seqrun.py: V12_SQ_SEEDS default 11, par 3) + feat_dw + dwbig/dw_s/mlp0-2/blend/write/validate; --cut 25 = local val mode). +- dwbig trained 1000 rounds (not 1200) since blend only uses round 1000; first 1000 trees identical. +- Verify run 1 (clean/raw symlinks only train.csv+evaluation.csv, work=sol_work/clean/work): stages setup..feat_pp, log clean/run1.log. Then compare matrix vs v12 cache, then --from feat_sq. +- 01:13 stages setup..feat_pp done 15.4 min (feat_tt 635 s). feat_sq+feat_dw running -> clean/run2.log + +## 01:40 Dublin - BOX RESTART incident +- Box rebooted ~01:20 (uptime 15 min at 01:35, journald 'uncleanly shut down'); /workspace was re-hydrated from a snapshot (all mtimes 01:20-01:27) WITHOUT any directory named `venv` or `out` (every out/ folder is gone: v12/out/submission_v12.csv, v2work/out/..., etc). Not caused by my clean test (only touched v12/sol_work/clean). +- venv rebuilt with uv at /workspace/venv (py3.13.5, pinned pkgs from sol/requirements.txt + pandas/pyarrow/duckdb/matplotlib). +- My verification run 2 (feat_sq) was killed by the restart (msC/s0 only partial). Features from stages setup..feat_pp survived in sol_work/clean/work. +- v12 shipped CSV regenerable from ens/{dwbig,dw_s,dwmlp}_c27 npy files (blend 1:1:2). +- 01:50 v12/out/submission_v12.csv RESTORED byte-identical (sha 37898b6f...10b) via recov/regen.py from ens/ npy (dwbig c1 + dw_s c1 + 2*mean dwmlp c3)/4, sigmoid, evaluation.csv order, header rowid,predicted_probability. Seq verification run 3 relaunched 01:43 (--from feat_sq --to feat_dw), 11 seeds. +- 01:38 probe built: probes/probe_v12_p33_w80.csv.zip sha 798cf0f0...7562; spearman(v12,p33)=0.9291 +- 02:23 clean run: feat_sq done in 2525s (11 seeds; eval cols corr .99-1.0 vs shipped sqC); feat_dw running +- 02:27 clean run: feat_dw 467s, eval cols corr 1.0 vs shipped. Launching --from dwbig --to validate (run3.log) +- 02:28 v11/out/submission_v11.csv regenerated from ens npy (sha f8bdd0a4..b0cb; no original full sha on record -> unverified; Spearman vs v12 .9938) +- 02:28 tools/package_final.py REF -> data/evaluation.csv (one-line change), gate re-tested: format pass, tampered rowid fails, low-AUC fails. Lost out/: v8 sub, v3 sub, v10 sub (sha bee2c7d5.. on record; from timed solution run, not regenerable w/o rerun), v13 (unshipped), v5/out results.jsonl+SEL.json, candidate zips. +- 02:42 CLEAN RUN c27 COMPLETE: out/submission_v12_sol.csv valid; Spearman vs shipped v12 = 0.99943 (pearson .99931, max|dp| .0507, mean .09350 vs .09335). Stage times sum 4635s = 77.3 min (8 threads): v3 prep 36, feats v3/v2/x/lr 220, tt 635, pp 35, seq(11 seeds) 2525, dw 467, dwbig 321, dw_s 74, mlp x3 321, blend/write/val 7. Fixed bug: --out dir now created (makedirs). stats in sol_work/verify_stats_c27.json. Launching --cut 25 in sol_work/clean25 (run1.log). +- 02:47 README_run.md written (c25 run in progress; prop features: mkprop.py ported at v12/prop/dw/mkprop.py, tables built in clean/work/feat_prop and clean25/work/feat_prop; matches DW cache_prop_c25 except tie-order in p_l3/p_streak for 0.03% rows) +- [03:52 UTC] c25 clean run DONE: LOCAL AUC cut25 dwbig .81661 dw_s .81611 mlp .81646 blend .81727 (v12 ref .81723, +.00004). Stats in sol_work/verify_stats_c25.json. Next: XGBoost check in /workspace/huawei/v12x (Eric request) BEFORE propensity. diff --git a/v12/sol/README_run.md b/v12/sol/README_run.md new file mode 100644 index 0000000000000000000000000000000000000000..eba3c016c4d1c5115568a301d004303fd7cb3d0a --- /dev/null +++ b/v12/sol/README_run.md @@ -0,0 +1,51 @@ +# solution_v12.py - Huawei TechArena 2026 Phase 1 (CTR), end-to-end from the raw host files + +One self-contained script: raw `train.csv` + `evaluation.csv` -> all features -> models -> `submission_v12.csv` +(columns `rowid,predicted_probability`, 529,390 rows, values in [0,1]). No absolute paths: everything lives under `--raw`, `--work`, `--out`. +All module sources are embedded in the script and written into `--work` at start-up. + +## What it does +1. v8/v10 feature families (v3 + v2 + extra similarity + lr2 + tt + pp), 377 columns. +2. `feat_sqC`: DIN-style sequence model refit at every fold cutoff and averaged over **11 seeds** (env `V12_SQ_SEEDS`), 20 stacking columns. +3. `feat_dw`: DW impression-taste (SVD of the user x item impression matrix), slate-centrality and skip-contrast features, 42 columns. +4. Models on folds f3..f6 (435 columns): LightGBM `dwbig` (1000 rounds), LightGBM `dw_s` (500 rounds), 3-seed MLP; logit blend 1:1:2, then sigmoid. +5. Writes and validates the CSV (row count, unique rowids, [0,1], no NaN). + +## Requirements +* Linux x86_64, **Python 3.13** (3.11+ should work), CPU only. `pip install -r requirements.txt` + (torch: `pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cpu`). +* About 12 GB RAM, about 40 GB free disk in the work dir, 8 cores recommended. +* Data: put `train.csv` (7.2 GB) and `evaluation.csv` together in one directory (the `--raw` dir). Nothing else is needed. + +## Run +```bash +python -m venv venv && . venv/bin/activate && pip install -r requirements.txt +python solution_v12.py --raw /path/to/data_dir --work /path/to/scratch --threads 8 --out /path/to/submission_v12.csv +``` +Stages are resumable: `--from STAGE --to STAGE` (stage list in the script docstring; finished stages are timed in `WORK/timings.json`). +Run it with `setsid nohup ... &` for the long run. + +Env knobs: `V12_SQ_SEEDS` (default 11; fewer seeds = faster and slightly weaker), `V12_SQ_PAR` (concurrent sequence runs, default min(3, threads//2)), +`V12_SQ_THREADS` (torch threads per sequence run, default 2). Set `OMP/MKL` thread vars yourself only if you know why; the script sets them per stage. + +## Expected runtime (8 threads, 8-core box, 15 GB RAM) +Verified end to end from a work dir holding only the two raw CSVs: **77.3 min** total (sum of stage times; the box restarted once mid-run, the sequence stage was redone in full). + +| stage | seconds | +|---|---| +| prep (v3 + v2) | 36 | +| feat v3 / v2 / x / lr | 220 | +| feat_tt | 635 | +| feat_pp | 35 | +| feat_sq (11 seeds, 3 concurrent x 2 threads) | 2525 | +| feat_dw | 467 | +| dwbig / dw_s | 321 / 74 | +| mlp seeds 0,1,2 | 126 / 95 / 100 | +| blend + write + validate | 7 | + +The sequence stage dominates (about 33 % of the time at 11 seeds); `V12_SQ_SEEDS=3` cuts it to about 12 min. + +## Verification result +* Output vs the shipped `submission_v12.csv`: Spearman **0.99943** (Pearson 0.99931, mean p 0.09350 vs 0.09335); seed noise only. +* Local check mode: `--cut 25` runs the same recipe on folds f1..f4 and scores `val25` (first event per user in Jan25-Jan27, labels from train); it prints/saves the AUC + in `WORK/local_auc_c25.json` and does not write a submission. Use a separate `--work` dir per cut. Shipped v12 local AUC at C25 = 0.81723. diff --git a/v12/sol/requirements.txt b/v12/sol/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fb274be23cf207323de0b52527bfa19c88ef70fe --- /dev/null +++ b/v12/sol/requirements.txt @@ -0,0 +1,7 @@ +# solution_v12.py - pinned to the versions it was verified with (Python 3.13.5, Linux x86_64, CPU only) +polars==2.0.0 +numpy==2.5.4 +scipy==1.18.1 +scikit-learn==1.9.1 +lightgbm==4.7.0 +torch==2.14.1 # CPU build: pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cpu diff --git a/v12/sol/solution_v12.py b/v12/sol/solution_v12.py new file mode 100644 index 0000000000000000000000000000000000000000..4d0ca357a344d1bf4422b887dae0a96ede04f2f8 --- /dev/null +++ b/v12/sol/solution_v12.py @@ -0,0 +1,77 @@ +"""Huawei TechArena 2026 Phase 1 - submission_v12: ONE self-contained end-to-end script (raw CSVs -> features -> models -> submission CSV). + +v12 recipe (local AUC: C25 0.81723 / C26 0.82604 / C23 0.79566; v10 was 0.8157 / 0.8247 / 0.7938 at C25 / C26 / C23): + features v8/v10 feature families (v3 + v2 + extra similarity + feat_lr2 + feat_tt + feat_pp; 'notime') -> 377 columns + + feat_sqC : DIN-style sequence model (seq11.py), refit at every fold cutoff, averaged over 11 seeds (0-10; seeds 8-10 use 4 heads/hid 128) + -> 20 stacking columns (logit, within-event z/dm/rank/gaps, attention stats, cosines, seed sd); 4 of them dropped (ens.py --drop) + + feat_dw : DW impression-taste (SVD of the user x item IMPRESSION matrix), slate-centrality and skip-contrast features -> 42 columns + = 435 model columns. All aggregates use train events with ts < fit cutoff only; each eval row only uses its own history and own slate. + models dwbig : LightGBM lr .015, 127 leaves, ff .5, l2 30, min_data 200, max_bin 255, negatives kept 100%, 1000 rounds, folds f3..f6 + dw_s : LightGBM lr .03, 95 leaves, ff .55, l2 30, max_bin 63, negatives kept 50% (seed 123), 500 rounds, folds f3..f6 + dwmlp : 512-256 MLP (rank-gauss inputs, 4 epochs, OneCycle), 3 seeds averaged, folds f3..f6 (negatives kept 50%) + blend z = (dwbig + dw_s + 2 * mean(dwmlp)) / 4 (logit blend 1:1:2), p = sigmoid(z) in [0, 1] + folds f_k = first event per user in [Jan20 + k d, + 1 d), k = 3..6 (labels known), stats frozen at that cutoff; eval features at Jan27. + +Usage: python solution_v12.py --raw DIR_WITH_train.csv_AND_evaluation.csv [--work WORKDIR] [--threads 8] [--out submission_v12.csv] + [--from STAGE] [--to STAGE] [--cut 27|25] + --cut 25 : local-validation mode (same recipe, folds f1..f4, scored on val25 = first event per user in [Jan25, Jan27) with the train labels; prints AUC). +Env knobs: V12_SQ_SEEDS (sequence seeds, default 11; e.g. 3 for a faster, slightly weaker run), V12_SQ_PAR (concurrent seq runs, default min(3, threads//2)), + V12_SQ_THREADS (torch threads per seq run), V12_MLP_SEEDS is fixed at 3. +Stages (resumable with --from/--to): setup prep_v3 prep_v2 feat_v3 feat_v2 feat_x feat_lr feat_tt feat_pp feat_sq feat_dw dwbig dw_s mlp0 mlp1 mlp2 blend write validate +Needs: python 3.11+ (tested 3.13), see requirements.txt (polars, numpy, scipy, scikit-learn, lightgbm, torch CPU). ~12 GB RAM, ~40 GB disk in WORKDIR. +The module sources are embedded below (SRC) and written to WORKDIR at start-up (no absolute paths anywhere; everything lives under --raw / --work / --out).""" +import os, sys, time, json, argparse, subprocess +SRC = {'avg_sq.py': '"""avg_sq.py OUT DIR1 DIR2 ... : average seq-model outputs across seed runs; event-relative columns recomputed from the averaged logit.\nRows are stored event-major (10 consecutive rows per event) in every dir; rowid order is asserted identical."""\nimport sys, os, numpy as np, polars as pl\nout, dirs = sys.argv[1], sys.argv[2:]; os.makedirs(out, exist_ok=True)\nnames = sorted(set.intersection(*[set(f[:-8] for f in os.listdir(d) if f.endswith(\'.parquet\')) for d in dirs]))\nfor n in names:\n T = [pl.read_parquet(f\'{d}/{n}.parquet\') for d in dirs]; rid = T[0][\'rowid\'].to_numpy()\n for t in T[1:]: assert (t[\'rowid\'].to_numpy() == rid).all(), (n, \'order\')\n cols = [c for c in T[0].columns if c != \'rowid\']\n M = {c: np.mean([t[c].to_numpy().astype(np.float64) for t in T], 0) for c in cols}\n S = M[\'sq_logit\'].reshape(-1, 10)\n M[\'sq_z\'] = ((S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6)).ravel(); M[\'sq_dm\'] = (S - S.mean(1, keepdims=True)).ravel()\n M[\'sq_lse\'] = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1).ravel()\n M[\'sq_rank\'] = (np.argsort(np.argsort(-S, 1, kind=\'stable\'), 1) + 1).ravel(); M[\'sq_gap\'] = (S - S.max(1, keepdims=True)).ravel()\n M[\'sq_gap2\'] = (S - np.sort(S, 1)[:, -2:-1]).ravel()\n M[\'sq_sd\'] = np.std([t[\'sq_logit\'].to_numpy() for t in T], 0) # seed disagreement\n df = pl.DataFrame({\'rowid\': rid, **M}).with_columns(pl.exclude(\'rowid\').cast(pl.Float32)); df.write_parquet(f\'{out}/{n}.parquet\'); print(n, df.shape, flush=True)\n', 'blend12.py': '"""v12 final blend: z = (dwbig@1000 + dw_s@500 + 2 * mean_over_3_seeds(dwmlp@epoch4)) / 4 (logit blend 1:1:2), p = sigmoid(z).\npython blend12.py CUT : CUT=27 -> eval_pred.parquet (rowid, p); CUT=25 -> prints/saves the local AUC on val25."""\nimport sys, json, numpy as np, polars as pl\nfrom cl import matrix, BASE\ncut = int(sys.argv[1]); vn = \'eval\' if cut == 27 else f\'val{cut}\'\n_, m, _ = matrix(vn, 1.0, extra=BASE)\nbig = np.load(f\'ens/dwbig_c{cut}_s0.npy\')[:, 0].astype(np.float64); sm = np.load(f\'ens/dw_s_c{cut}_s0.npy\')[:, 0].astype(np.float64)\nml = np.mean([np.load(f\'ens/dwmlp_c{cut}_s{s}.npy\')[:, 3].astype(np.float64) for s in (0, 1, 2)], 0)\nassert len(big) == len(sm) == len(ml) == m.height\nz = (big + sm + 2 * ml) / 4; p = 1 / (1 + np.exp(-z)); assert np.isfinite(p).all() and (p >= 0).all() and (p <= 1).all()\nprint(\'corr big/small/mlp\', round(float(np.corrcoef(big, sm)[0, 1]), 4), round(float(np.corrcoef(big, ml)[0, 1]), 4))\nif cut == 27:\n pl.DataFrame({\'rowid\': m[\'rowid\'], \'p\': p}).write_parquet(\'eval_pred.parquet\'); print(\'wrote eval_pred.parquet\', m.height)\nelse:\n from sklearn.metrics import roc_auc_score\n y = m[\'clicked\'].to_numpy(); r = {k: round(roc_auc_score(y, v), 5) for k, v in dict(dwbig=big, dw_s=sm, mlp=ml, blend=z).items()}\n print(\'LOCAL AUC cut\', cut, r); json.dump(r, open(f\'local_auc_c{cut}.json\', \'w\'))\n', 'build_folds.py': "import polars as pl, sys, os\nsys.path.insert(0,'@@WORK@@')\nfrom feats import load, first_events, build, log\nD='@@WORK@@'; os.makedirs(f'{D}/feat',exist_ok=True)\nTE,TR,EE,ER,NI=load(D)\nDAY=86400; J20=1737331200\nEND=J20+7*DAY; VC=J20+4*DAY\n# validation: V = first event/user in [Jan24,Jan27); training folds end before Jan24\nspecs=[(VC,END-VC,'V')]+[(J20+k*DAY,VC-(J20+k*DAY),f'g{k}') for k in (1,2,3)]\nif len(sys.argv)>1 and sys.argv[1]=='c25':\n specs=[(J20+5*DAY,2*DAY,'val25')]+[(J20+k*DAY,DAY,f'h{k}') for k in (1,2,3,4)]\nif len(sys.argv)>1 and sys.argv[1]=='c26':\n specs=[(J20+6*DAY,DAY,'val26')]+[(J20+k*DAY,DAY,f'h{k}') for k in (5,)]+[('eval',None,'eval')]\nif len(sys.argv)>1 and sys.argv[1]=='eft':\n specs=[(J20+k*DAY,(J20+5*DAY)-(J20+k*DAY),f'e{k}') for k in (1,2,3,4)]+[(J20+k*DAY,END-(J20+k*DAY),f'E{k}') for k in (1,2,3,4,5,6)]\nif len(sys.argv)>1 and sys.argv[1]=='dft':\n specs=[(J20+k*DAY,(J20+6*DAY)-(J20+k*DAY),f'd{k}') for k in (1,2,3,4,5)]\nif len(sys.argv)>1 and sys.argv[1]=='final':\n specs=[(J20+k*DAY,END-(J20+k*DAY),f'F{k}') for k in (1,2,3,4,5,6)]+[('eval',None,'eval')]\nif len(sys.argv)>1 and sys.argv[1]=='lite': # v4_lite: only what the final fit needs (h1..h5 + val26 = f1..f6, eval)\n specs=[(J20+k*DAY,DAY,f'h{k}') for k in (3,4,5)]+[(J20+6*DAY,DAY,'val26'),('eval',None,'eval')]\nfor C,W,name in specs:\n out=f'{D}/feat/{name}.parquet'\n if os.path.exists(out): continue\n if C=='eval':\n C=J20+7*DAY; T_ev=EE; T_rows=ER\n else:\n T_ev=first_events(TE,C,W); T_rows=TR.join(T_ev.select('eid'),on='eid')\n R=build(C,T_ev,T_rows,TE,TR,NI)\n R.drop('h','h_ts').write_parquet(out)\n log('wrote',name,R.shape)\n", 'cl.py': '"""Shared loader + feature-matrix cache (step C). Matrices cached as parquet keyed by hash(table names, feature list, sample)."""\nimport os, sys, hashlib, json, gc, numpy as np, polars as pl\nsys.path.insert(0,\'@@WORK@@\'); sys.path.insert(1,\'@@WORK@@/v3\')\nfrom feats import DROP\nfrom features import feature_cols, GROUPS\nD=\'@@WORK@@\'; P2=f\'{D}/feat\'; P3=f\'{D}/feat_v3\'; CACHE=f\'{D}/cache_mx\'; os.makedirs(CACHE,exist_ok=True)\nV2_OF={\'f1\':\'h1\',\'f2\':\'h2\',\'f3\':\'h3\',\'f4\':\'h4\',\'f5\':\'h5\',\'f6\':\'val26\',\'val25\':\'val25\',\'val26\':\'val26\',\'eval\':\'eval\'}\nV2_OF.update({f\'{p}{k}\':f\'{p}{k}\' for p in \'eEd\' for k in range(1,7)})\nTIME=[\'tm_stale\',\'tm_hour\',\'tm_tb\',\'tm_dt\',\'v2_tb\',\'v2_dt\',\'v2_stale\',\'v2_hour\']\nKEEP=[\'rowid\',\'eid\',\'u\',\'ts\',\'clicked\',\'us_cold\',\'us_nohist\',\'tm_stale\']\ndef load_tab(n, extra=(), variant=\'notime\'):\n a=pl.read_parquet(f\'{P3}/{n}.parquet\'); b=pl.read_parquet(f\'{P2}/{V2_OF[n]}.parquet\')\n f2=[c for c in b.columns if c not in DROP]\n a=a.join(b.select([\'rowid\']+f2).rename({c:\'v2_\'+c for c in f2}),on=\'rowid\',how=\'left\')\n for sub in (os.environ.get(\'XDIR\',\'feat_x\'),)+((\'feat_x2\',) if os.path.isdir(f\'{D}/feat_x2\') else ())+tuple(extra):\n a=a.join(pl.read_parquet(f\'{D}/{sub}/{n}.parquet\'),on=\'rowid\',how=\'left\')\n if \'+\' in variant:\n raise ValueError(\'mods not used in v7\')\n return a\ndef featlist(df, variant=\'notime\'):\n f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith((\'v2_\',\'x_\',\'r_\',\'s_\',\'ev2_\',\'rc_\',\'sh_\',\'hs_\',\'lr\',\'tA_\',\'tB_\'))]\n if \'notime\' in variant: f=[c for c in f if c not in TIME]\n if variant.split(\'+\')[0]==\'stale\': f=[c for c in f if c not in TIME or c==\'tm_stale\']\n return f\nBASE=(\'feat_lr2\',\'feat_tt\',\'feat_pp\')\ndef matrix(n, frac=1.0, variant=\'notime\', extra=()):\n """v12: base = v8-lite matrix (v2/v3/x + feat_lr2/feat_tt/feat_pp, \'notime\'), cached as parquet; every other dir in `extra`\n (feat_sqC, feat_dw) is left-joined on rowid and its columns appended. returns (X float32, meta DataFrame, feats)."""\n base=tuple(e for e in extra if e in BASE); new=[e for e in extra if e not in BASE]; assert base==BASE, extra\n key=hashlib.md5(json.dumps([n,variant,list(base),[os.path.getmtime(f) for f in (f\'{P3}/{n}.parquet\',f\'{P2}/{V2_OF[n]}.parquet\',f\'{D}/feat_x/{n}.parquet\')+tuple(f\'{D}/{b}/{n}.parquet\' for b in base)]]).encode()).hexdigest()[:12]\n fp=f\'{CACHE}/{n}_{key}.parquet\'\n if os.path.exists(fp): t=pl.read_parquet(fp); fl=json.load(open(fp+\'.json\'))\n else:\n t=load_tab(n,base,variant); fl=featlist(t,variant)\n t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl)))\n t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP])\n t.write_parquet(fp+\'.tmp\'); os.replace(fp+\'.tmp\',fp); json.dump(fl,open(fp+\'.json\',\'w\'))\n for sub in new:\n e=pl.read_parquet(f\'{D}/{sub}/{n}.parquet\'); nc=[c for c in e.columns if c!=\'rowid\' and c not in fl]\n t=t.join(e.select([\'rowid\']+nc),on=\'rowid\',how=\'left\',maintain_order=\'left\'); fl=fl+nc\n X=t.select(fl).cast(pl.Float32).to_numpy(); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect()\n return X, meta, fl\n', 'dw/common.py': '"""DW signals shared loader (vendored from dw_signals/common.py; paths -> work dir).\nSnapshot(C): everything known from train events with ts < C (rows+labels of those events, plus click log reconstructed from their histories).\n"""\nimport numpy as np, polars as pl, time, gc\nfrom scipy.sparse import csr_matrix\nfrom scipy.sparse.linalg import svds\nimport os\nL=os.environ.get(\'DW_WORK\',\'@@WORK@@\'); DAY=86400; J20=1737331200 # L = work dir holding train_events/train_rows/evaluation_* parquet (prep.py output)\ndef Jd(d): return J20+(d-20)*DAY\nT0=time.time()\ndef log(*a): print(f\'[{time.time()-T0:6.0f}s]\',*a,flush=True)\n\nclass Data:\n def __init__(s):\n ev=pl.read_parquet(f\'{L}/train_events.parquet\').sort(\'ts\').with_row_index(\'e\')\n rows=pl.read_parquet(f\'{L}/train_rows.parquet\').join(ev.select(\'eid\',\'e\'),on=\'eid\').sort([\'e\',\'item\'])\n s.NE=ev.height; assert rows.height==s.NE*10\n s.ETS=ev[\'ts\'].to_numpy(); s.EID=ev[\'eid\'].to_numpy()\n uu,s.EU=np.unique(ev[\'user\'].to_numpy(),return_inverse=True); s.EU=s.EU.astype(np.int32); s.NU=len(uu); s.uu=uu\n s.Hlen=ev[\'h_items\'].list.len().to_numpy().astype(np.int64)\n s.OFF=np.zeros(s.NE+1,np.int64); s.OFF[1:]=np.cumsum(s.Hlen)\n hi=ev[\'h_items\'].explode().drop_nulls().to_numpy(); s.HT=ev[\'h_ts\'].explode().drop_nulls().to_numpy().astype(np.int64)\n rr=rows[\'item\'].to_numpy()\n s.items=np.unique(np.concatenate([np.unique(rr),np.unique(hi)])); s.NI=len(s.items)\n s.HI=np.searchsorted(s.items,hi).astype(np.int32); s.RI=np.searchsorted(s.items,rr).astype(np.int32).reshape(s.NE,10)\n s.Y=rows[\'clicked\'].to_numpy().astype(np.int8).reshape(s.NE,10)\n s.rowid=rows[\'rowid\'].to_numpy().reshape(s.NE,10)\n s.eid2e={} # not needed\n s.ev=ev.select(\'e\',\'eid\',\'user\',\'ts\'); s.rows_rowid=rows.select(\'rowid\',\'e\',\'item\')\n del ev,rows; gc.collect()\n def snap(s,X):\n """click log (user,item,ts) from events with ts>20; tm=key&((1<<20)-1)\n return dict(nX=nX,cu=(ui//s.NI).astype(np.int32),ci=(ui%s.NI).astype(np.int32),ct=tm*60+Jd(1))\n\ndef click_svd(S,D,k=32):\n """item/user taste vectors from dedup click log (binary, 1/sqrt((nu+1)(ni+1)) scaling). returns item vec (V*s) L2-normed, user vec"""\n u,i=S[\'cu\'],S[\'ci\']; key=np.unique(u.astype(np.int64)*D.NI+i); uu=(key//D.NI).astype(np.int32); ii=(key%D.NI).astype(np.int32)\n nu=np.bincount(uu,minlength=D.NU).astype(np.float32); ni=np.bincount(ii,minlength=D.NI).astype(np.float32)\n M=csr_matrix((1/np.sqrt((nu[uu]+1)*(ni[ii]+1)),(uu,ii)),shape=(D.NU,D.NI),dtype=np.float32)\n U,sv,Vt=svds(M,k=k,random_state=0)\n V=(Vt.T*sv); V/= np.linalg.norm(V,axis=1,keepdims=True)+1e-9\n return V.astype(np.float32),(U*sv).astype(np.float32)\ndef norm(x): return x/(np.linalg.norm(x,axis=-1,keepdims=True)+1e-9)\ndef event_view(a): return a.reshape(-1,10)\ndef within_event(x):\n """z-score, demeaned, rank (0..9 ascending) within the 10-row events; x aligned with val rows"""\n m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6\n rk=m.argsort(1).argsort(1).astype(np.float32)\n return ((m-mu)/sd).ravel(),(m-mu).ravel(),rk.ravel()\n', 'dw/mkdw.py': '"""mkdw.py NAME... : DW reusable feature families (impression_taste, slate_centrality, skip_contrast) for rows of fold/val/eval table NAME.\nFit cutoff X: f{k} -> J20+k*DAY; val23/25/26 -> J20+3/5/6 d; eval -> J27 (end of train). Only train events ts one empty extra user row."""\n ee=pl.read_parquet(f\'{W}/evaluation_events.parquet\').sort(\'ts\',\'eid\'); er=pl.read_parquet(f\'{W}/evaluation_rows.parquet\')\n assert ee[\'ts\'].min()>=D.ETS.max()\n n=ee.height; eid2e={x:i for i,x in enumerate(ee[\'eid\'].to_list())}\n er=er.with_columns(pl.col(\'eid\').replace_strict(eid2e).alias(\'_e\')).sort(\'_e\',\'rowid\'); assert (np.bincount(er[\'_e\'].to_numpy())==10).all()\n pos=np.searchsorted(D.uu,ee[\'user\'].to_numpy()); pos=np.minimum(pos,D.NU-1); seen=D.uu[pos]==ee[\'user\'].to_numpy()\n new_u=D.NU; eu=np.where(seen,pos,new_u).astype(np.int32); print(\'eval unseen users\',int((~seen).sum()),flush=True)\n ri=np.searchsorted(D.items,er[\'item\'].to_numpy()); assert (D.items[ri]==er[\'item\'].to_numpy()).all(); ri=ri.astype(np.int32)\n hi=ee[\'h_items\'].explode().drop_nulls().to_numpy(); ht=ee[\'h_ts\'].explode().drop_nulls().to_numpy().astype(np.int64); hl=ee[\'h_items\'].list.len().to_numpy()\n hp=np.minimum(np.searchsorted(D.items,hi),D.NI-1); ok=D.items[hp]==hi; print(\'eval hist items unseen (dropped)\',int((~ok).sum()),\'of\',len(hi),flush=True)\n hl=hl.astype(np.int64); cs=np.concatenate([[0],np.cumsum(ok.astype(np.int64))]); ce=np.concatenate([[0],np.cumsum(hl)]); hl2=cs[ce[1:]]-cs[ce[:-1]]\n D0=D.NE; D.ETS=np.concatenate([D.ETS,ee[\'ts\'].to_numpy()]); D.EU=np.concatenate([D.EU,eu]); D.Hlen=np.concatenate([D.Hlen,hl2.astype(np.int64)])\n base=D.OFF[-1]; D.OFF=np.concatenate([D.OFF[:-1],base+np.concatenate([[0],np.cumsum(hl2)])]).astype(np.int64)\n D.HI=np.concatenate([D.HI,hp[ok].astype(np.int32)]); D.HT=np.concatenate([D.HT,ht[ok]])\n D.RI=np.concatenate([D.RI,ri.reshape(n,10)]); D.Y=np.concatenate([D.Y,np.zeros((n,10),np.int8)]); D.NE=D0+n; D.NU=D.NU+1\n assert len(D.OFF)==D.NE+1\n e=np.repeat(np.arange(D0,D0+n),10)\n return dict(rowid=er[\'rowid\'].to_numpy(),e=e,item=ri,user=D.EU[e],ts=D.ETS[e])\nif __name__==\'__main__\':\n D=Data(); names=sys.argv[1:]\n for nm in names:\n t=time.time(); X=CUT[nm]\n if nm==\'eval\':\n nt=D.NE; V=extend_eval(D); assert int(np.searchsorted(D.ETS,X))==nt # snapshot = exactly the train events\n else: V=rows_train(D,nm)\n df=all_recommended(D,X,V); df=df.with_columns(pl.Series(\'rowid\',V[\'rowid\'])).select([\'rowid\']+[c for c in df.columns])\n df.write_parquet(f\'{OUT}/{nm}.parquet\'); s=round(time.time()-t); print(nm,df.shape,f\'{s}s\',flush=True)\n', 'dw/reusable.py': '"""Clean entry points for the recommended new signals.\n\nAll functions take\n D : common.Data (lite events/rows loader)\n X : fit cutoff, unix seconds (only train events with ts < X are used for any fit/aggregate)\n rows : dict from D.val(c) / same layout: \'user\',\'item\',\'ts\',\'e\' aligned to n_events*10 rows (10 candidates per event)\nand return {feature_name: float32 array of len(rows[\'item\'])} (NaN = no information for that row).\n\nLeak-safety: every statistic (impression matrix, SVDs, click log) is built from events with ts < X only; the only\nper-row inputs are the row\'s own user id, candidate item, ts and the row\'s OWN pre-ts history (D.HI/HT/OFF). No eval-row\nlabels, impressions or histories are ever aggregated. Rows\' within-slate stats (centrality) use only the row\'s own slate.\n\n from reusable import impression_taste, slate_centrality, skip_contrast, all_recommended\n"""\nimport numpy as np, polars as pl\nfrom common import *\nimport signals as S_\n\ndef impression_taste(D,X,rows,ks=(8,16,32,64,128)):\n """#1 Impression-SVD taste. SVD(k) of the user x item IMPRESSION-count matrix (train events tspl.DataFrame:\n F={}; F.update(impression_taste(D,X,rows)); F.update(slate_centrality(D,X,rows)); F.update(skip_contrast(D,X,rows))\n return pl.DataFrame({k:np.asarray(v,np.float32) for k,v in F.items()})\n\nif __name__==\'__main__\':\n import sys; c=int(sys.argv[1]); D=Data(); V=D.val(c); df=all_recommended(D,Jd(c),V); print(df.shape); print(df.columns)\n', 'dw/signals.py': '"""Candidate signals. Each function: f(D, S, V, X) -> dict name->float32 array aligned to val rows (len n*10), using ONLY snapshot S (train events ts=X-3*DAY)),u,it,y,t\n\ndef hist_flat(D,V):\n """own-history items/ts of each val event, flattened, with row->event pointer"""\n e=V[\'e\'][::10]; ln=D.Hlen[e]; off=np.zeros(len(e)+1,np.int64); off[1:]=np.cumsum(ln)\n idx=np.concatenate([np.arange(D.OFF[x],D.OFF[x+1]) for x in e]) if len(e) else np.array([],int)\n return D.HI[idx],D.HT[idx],off,ln\n\ndef sig_impression_svd(D,S,V,X,k=32,Mimp=None):\n """A. taste embedding from IMPRESSIONS (who was shown what), not clicks: SVD of user x item impression-count matrix."""\n nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X)\n nu=np.asarray(Mi.sum(1)).ravel(); ni=np.asarray(Mi.sum(0)).ravel()\n r,c=Mi.nonzero(); w=np.asarray(Mi[r,c]).ravel()\n M=csr_matrix((w/np.sqrt((nu[r]+1)*(ni[c]+1)),(r,c)),shape=Mi.shape,dtype=np.float32)\n U,sv,Vt=svds(M,k=k,random_state=0); Vi=(Vt.T*sv).astype(np.float32); Us=(U*sv).astype(np.float32)\n u=V[\'user\']; it=V[\'item\']; has=(nu[u]>0)\n rec=np.einsum(\'ij,ij->i\',U[u]*np.sqrt(sv),Vt.T[it]*np.sqrt(sv)); \n cos=np.einsum(\'ij,ij->i\',norm(Us[u]),norm(Vi[it]))\n out=dict(impsvd_rec=rec,impsvd_cos=cos); \n for k_,v_ in out.items(): v_[~has]=np.nan\n z=within_event(cos); out[\'impsvd_cos_z\']=z[0]; out[\'impsvd_cos_rk\']=z[2]\n return {k_:v_.astype(np.float32) for k_,v_ in out.items()},dict(Vi=Vi)\n\ndef sig_skip_embedding(D,S,V,X,Vclk):\n """B. negative feedback in embedding space: centroid of items the user was SHOWN but did NOT click (all train before X / last 3d) and centroid of all shown items, cos with candidate (click-SVD item vectors),\n plus own-history centroid cos (reference) and skip-minus-history contrasts."""\n nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X)\n u=V[\'user\']; it=V[\'item\']; Ec=Vclk[it]\n def cen(M):\n s=M@Vclk; n=np.asarray(M.sum(1)).ravel(); return s, n\n out={}\n for nm,M in ((\'skall\',Ms),(\'skrec\',Mr),(\'impall\',Mi)):\n s,n=cen(M); cs=norm(s)[u]; c=np.einsum(\'ij,ij->i\',cs,Ec).astype(np.float32); c[n[u]==0]=np.nan; out[\'c_\'+nm]=c\n hi,ht,off,ln=hist_flat(D,V)\n hs=np.add.reduceat(np.vstack([Vclk[hi],np.zeros((1,Vclk.shape[1]),np.float32)]),np.minimum(off[:-1],len(hi)),axis=0) if len(hi) else None\n hs=np.where((ln>0)[:,None],hs,0); hc=np.repeat(norm(hs),10,axis=0)\n c=np.einsum(\'ij,ij->i\',hc,Ec).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out[\'c_hist\']=c\n out[\'c_hist_minus_sk\']=out[\'c_hist\']-out[\'c_skall\']; out[\'c_hist_minus_skrec\']=out[\'c_hist\']-out[\'c_skrec\']\n # skip-centroid of the user\'s LAST event slate only (non-clicked), what the user just rejected\n return out,{}\n\ndef sig_slate_centrality(D,S,V,X,Vemb,tag=\'clk\'):\n """C. slate coherence: candidate\'s mean / max cosine to the other 9 candidates in the same slate (odd-one-out / retrieved-cluster membership)."""\n E=Vemb[V[\'item\']].reshape(-1,10,Vemb.shape[1]); G=np.einsum(\'nik,njk->nij\',E,E); idx=np.arange(10); G[:,idx,idx]=np.nan\n mean=np.nanmean(G,2); mx=np.nanmax(G,2); top3=np.sort(np.nan_to_num(G,nan=-9),2)[:,:,-3:].mean(2)\n out={f\'cent_{tag}_mean\':mean.ravel(),f\'cent_{tag}_max\':mx.ravel(),f\'cent_{tag}_top3\':top3.ravel()}\n out[f\'cent_{tag}_mean_dm\']=(mean-mean.mean(1,keepdims=True)).ravel()\n out[f\'slate_{tag}_coh\']=np.repeat(mean.mean(1),10)\n return {k:v.astype(np.float32) for k,v in out.items()},{}\n\ndef sig_exposure_lift(D,S,V,X):\n """E. Pearson-residual of user-item exposure vs independence (u_imp * i_imp / N); same for skipped-only impressions."""\n nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X); N=Mi.sum()\n u=V[\'user\']; it=V[\'item\']; ui=np.asarray(Mi[u,it]).ravel(); us=np.asarray(Ms[u,it]).ravel()\n nu=np.asarray(Mi.sum(1)).ravel()[u]; ni=np.asarray(Mi.sum(0)).ravel()[it]\n ex=nu*ni/N; out={\'expo_resid\':(ui-ex)/np.sqrt(ex+0.05),\'expo_ratio\':np.log((ui+0.5)/(ex+0.5)),\'skip_cnt\':us.astype(np.float32),\'skip_frac\':(us+0.0)/(ui+1)}\n return {k:v.astype(np.float32) for k,v in out.items()},{}\n\ndef sig_coimpression(D,S,V,X):\n """F. slate co-membership: for candidate c and each own-history item h, how often h and c were shown in the same train slate (before X), as lift over independence; sum/mean/max over h."""\n nX=int(np.searchsorted(D.ETS,X)); R=D.RI[:nX]\n B=csr_matrix((np.ones(nX*10,np.float32),(np.repeat(np.arange(nX),10),R.ravel())),shape=(nX,D.NI))\n C=(B.T@B).tocsr(); C.setdiag(0); C.eliminate_zeros(); ni=np.asarray(B.sum(0)).ravel()\n keys=None\n hi,ht,off,ln=hist_flat(D,V); n=len(V[\'e\'])//10\n evi=np.repeat(np.arange(n),ln) # event of each history entry\n # all (hist item, candidate) pairs: for each event, 20x10\n pe=np.repeat(evi,10); ph=np.repeat(hi,10); pc=V[\'item\'].reshape(n,10)[evi].ravel(); prow=(np.repeat(evi,10)*10+np.tile(np.arange(10),len(evi)))\n cnt=np.asarray(C[ph,pc]).ravel().astype(np.float32)\n Nimp=nX*10.0; ex=ni[ph]*ni[pc]*9.0/Nimp # P(two given impressions share a slate)=9/N_imp\n lift=(cnt-ex)/np.sqrt(ex+0.05)\n out={}\n for nm,val in ((\'sum\',cnt),(\'lift\',lift)):\n s=np.bincount(prow,weights=val,minlength=n*10); out[\'coimp_\'+nm+\'_sum\']=s\n out[\'coimp_\'+nm+\'_mean\']=s/np.maximum(np.repeat(ln,10),1)\n mx=np.full(n*10,-1e9,np.float32); np.maximum.at(mx,prow,lift); mx[mx<-1e8]=np.nan; out[\'coimp_lift_max\']=mx\n return {k:np.asarray(v,np.float32) for k,v in out.items()},{}\n\ndef sig_user_time(D,S,V,X):\n """H+D. user-level time/behaviour: hour-of-day match of click times / event times, last-event outcome, recent event click rate, same-time-bucket event click rate."""\n nX=int(np.searchsorted(D.ETS,X)); u=V[\'user\']; t=V[\'ts\']; n=len(u)//10\n hr=lambda ts: ((ts%DAY)//3600).astype(np.int64)\n # click hours per user from click log\n Hc=np.zeros((D.NU,24),np.float32); np.add.at(Hc,(S[\'cu\'],hr(S[\'ct\'])),1)\n He=np.zeros((D.NU,24),np.float32); np.add.at(He,(D.EU[:nX],hr(D.ETS[:nX])),1)\n h=hr(t)\n def win(H):\n w=sum(H[u,(h+d)%24] for d in (-2,-1,0,1,2)); tot=H[u].sum(1); return ((w+1)/(tot+24/5.0)/(5/24)).astype(np.float32),tot\n out={}; out[\'hr_click_match\'],tc=win(Hc); out[\'hr_ev_match\'],te=win(He)\n # last-event outcome / recent click rate / same-bucket ctr (event-level)\n evclk=D.Y[:nX].sum(1); eu=D.EU[:nX]; ets=D.ETS[:nX]; tb=((ets%DAY)//21600)\n order=np.lexsort((ets,eu)); eus=eu[order]; \n last_e=np.full(D.NU,-1,np.int64)\n # vectorised: last event index per user = max index among that user\'s events\n np.maximum.at(last_e,eu,np.arange(nX))\n lc=np.where(last_e[u]>=0,evclk[np.maximum(last_e[u],0)],np.nan); out[\'last_ev_clicked\']=lc.astype(np.float32)\n # recent-5 event click rate\n rank_in_user=np.empty(nX,np.int64); rank_in_user[order]=np.arange(nX)-np.searchsorted(eus,eus)\n cnt=np.bincount(eu,minlength=D.NU); frm_end=cnt[eu]-1-rank_in_user # 0 = last\n m5=frm_end<5; s5=np.bincount(eu[m5],weights=evclk[m5],minlength=D.NU); n5=np.bincount(eu[m5],minlength=D.NU)\n out[\'ev_ctr_last5\']=((s5[u]+0.629*2)/(n5[u]+2)).astype(np.float32)\n Sb=np.zeros((D.NU,4),np.float32); Nb=np.zeros((D.NU,4),np.float32); np.add.at(Sb,(eu,tb),evclk); np.add.at(Nb,(eu,tb),1)\n tbv=(h//6); out[\'ev_ctr_samebucket\']=((Sb[u,tbv]+0.629*3)/(Nb[u,tbv]+3)).astype(np.float32)\n out[\'ev_cnt_samebucket_frac\']=((Nb[u,tbv]+0.5)/(Nb[u].sum(1)+2)).astype(np.float32)\n return out,{}\n\ndef _svd_scaled(M,k):\n nu=np.asarray(M.sum(1)).ravel(); ni=np.asarray(M.sum(0)).ravel(); M=M.tocoo()\n Ms=csr_matrix((M.data/np.sqrt((nu[M.row]+1)*(ni[M.col]+1)),(M.row,M.col)),shape=M.shape,dtype=np.float32)\n U,sv,Vt=svds(Ms,k=k,random_state=0); return (U*sv).astype(np.float32),(Vt.T*sv).astype(np.float32),nu\n\ndef sig_impression_variants(D,S,V,X):\n """A2. variants of the impression-taste embedding: other ranks, skip-only SVD, joint click+impression SVD, recent-impression centroid, and the user\'s click-history centroid measured in IMPRESSION item space."""\n nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X); u=V[\'user\']; it=V[\'item\']; out={}\n def ucos(U_,V_,nu,tag):\n c=np.einsum(\'ij,ij->i\',norm(U_[u]),norm(V_[it])).astype(np.float32); c[nu[u]==0]=np.nan; out[tag]=c; out[tag+\'_z\']=within_event(c)[0]\n for k in (16,64):\n U_,V_,nu=_svd_scaled(Mi,k); ucos(U_,V_,nu,f\'impsvd{k}_cos\')\n U_,V_,nu=_svd_scaled(Ms,32); ucos(U_,V_,nu,\'skipsvd32_cos\')\n # joint click(log, dedup)+impression\n ck=csr_matrix((np.ones(len(S[\'cu\']),np.float32),(S[\'cu\'],S[\'ci\'])),shape=(D.NU,D.NI)); ck.data[:]=1\n from scipy.sparse import hstack\n U_,V_,nu=_svd_scaled(hstack([ck*3.0,Mi]).tocsr(),32); Vj=V_[:D.NI] # click-part item vecs, joint user space\n c=np.einsum(\'ij,ij->i\',norm(U_[u]),norm(Vj[it])).astype(np.float32); out[\'joint32_cos_clkpart\']=c; out[\'joint32_cos_clkpart_z\']=within_event(c)[0]\n Vimp=_svd_scaled(Mi,32)[1]; Vn=norm(Vimp)\n # own-history centroid in impression item space\n hi,ht,off,ln=hist_flat(D,V); n=len(ln)\n hs=np.add.reduceat(np.vstack([Vn[hi],np.zeros((1,32),np.float32)]),np.minimum(off[:-1],len(hi)),axis=0); hs=np.where((ln>0)[:,None],hs,0)\n c=np.einsum(\'ij,ij->i\',np.repeat(norm(hs),10,axis=0),Vn[it]).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out[\'hist_cos_impspace\']=c; out[\'hist_cos_impspace_z\']=within_event(c)[0]\n # recent (3d) impression centroid in click space needs Vclk -> in compute.py\n return out,{}\n\ndef sig_recent_impression(D,S,V,X,Vclk,days=(1,3)):\n nX=int(np.searchsorted(D.ETS,X)); u=V[\'user\']; it=V[\'item\']; out={}\n for d in days:\n m=D.ETS[:nX]>=X-d*DAY; r=np.nonzero(m)[0]\n M=csr_matrix((np.ones(len(r)*10,np.float32),(np.repeat(D.EU[r],10),D.RI[r].ravel())),shape=(D.NU,D.NI)); n=np.asarray(M.sum(1)).ravel()\n c=np.einsum(\'ij,ij->i\',norm(M@Vclk)[u],Vclk[it]).astype(np.float32); c[n[u]==0]=np.nan; out[f\'c_imp{d}d\']=c\n # decayed by event: also impression count in window of this exact item\n out[f\'ui_imp{d}d\']=np.asarray(M[u,it]).ravel().astype(np.float32)\n return out,{}\n\ndef sig_impression_multirank(D,S,V,X,ks=(8,64,128)):\n """A3. impression-SVD at more ranks: user-cos, plus own-click-history centroid cos and max-sim (over history items) in that impression item space."""\n nX,Mi,*_=_imp_mats(D,X); u=V[\'user\']; it=V[\'item\']; out={}\n hi,ht,off,ln=hist_flat(D,V); n=len(ln); evi=np.repeat(np.arange(n),ln); itm=it.reshape(n,10)\n for k in ks:\n U_,V_,nu=_svd_scaled(Mi,k); Vn=norm(V_)\n if k in (8,128): c=np.einsum(\'ij,ij->i\',norm(U_[u]),Vn[it]).astype(np.float32); c[nu[u]==0]=np.nan; out[f\'impsvd{k}_cos\']=c\n hs=np.zeros((n,k),np.float32); np.add.at(hs,evi,Vn[hi]); c=np.einsum(\'ij,ij->i\',np.repeat(norm(hs),10,axis=0),Vn[it]).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out[f\'hist_cos_imp{k}\']=c\n if k==64: # max-sim over history items\n mx=np.full((n,10),-1.0,np.float32); H=Vn[hi]\n for j in range(10):\n s=np.einsum(\'ik,ik->i\',H,Vn[itm[evi,j]]); np.maximum.at(mx[:,j],evi,s)\n mx=mx.ravel(); mx[np.repeat(ln==0,10)]=np.nan; out[\'hist_maxsim_imp64\']=mx\n for kk in list(out): out[kk+\'_z\']=within_event(np.nan_to_num(out[kk],nan=np.nanmedian(out[kk])))[0]\n return out,{}\n', 'ens.py': '"""LightGBM component of the v12 recipe. python ens.py CUT TAG [--seeds 0] [--drop re] [--params json] [--rounds N] [--save r1,r2] [--lr dirs] [--nfolds 4] [--negfrac f]\nCUT=27: train f(CUT-24)..f(CUT-21) (f3..f6), predict eval; CUT=25: train f1..f4, predict val25 (AUC printed).\nNegatives of the concatenated training folds are subsampled with one RandomState(123).rand(total) draw. Saves ens/{tag}_c{CUT}_s{seed}.npy (raw scores at the --save rounds)."""\nimport sys, os, re, json, gc, time, argparse, numpy as np, lightgbm as lgb, polars as pl\nfrom cl import matrix\nfrom sklearn.metrics import roc_auc_score\nap=argparse.ArgumentParser(); ap.add_argument(\'cut\',type=int); ap.add_argument(\'tag\'); ap.add_argument(\'--seeds\',default=\'0\')\nap.add_argument(\'--drop\',default=None); ap.add_argument(\'--params\',default=\'{}\'); ap.add_argument(\'--rounds\',type=int,default=400); ap.add_argument(\'--threads\',type=int,default=8)\nap.add_argument(\'--lr\',default=\'\'); ap.add_argument(\'--nfolds\',type=int,default=4); ap.add_argument(\'--negfrac\',type=float,default=1.0); ap.add_argument(\'--save\',default=\'400\')\na=ap.parse_args(); t0=time.time(); os.makedirs(\'ens\',exist_ok=True)\nextra=tuple(x for x in a.lr.split(\',\') if x)\ntrn=[f\'f{k}\' for k in range(max(1,a.cut-20-a.nfolds),a.cut-20)]\nlens=[len(pl.read_parquet(f\'feat_v3/{n}.parquet\',columns=[\'rowid\'])) for n in trn]\nR=np.random.RandomState(123).rand(sum(lens)) if a.negfrac<1 else None\nXs=[];ys=[];feats=None;off=0\nfor n,L in zip(trn,lens):\n X,m,f=matrix(n,1.0,extra=extra); feats=f; yy=m[\'clicked\'].to_numpy(); assert len(yy)==L\n sel=(yy==1)|(R[off:off+L] v2work/feat_x/NAME.parquet\n - con variants: k=32/128, recency-weighted (tau=2d) click/non-click centroids, item-side con (users who clicked i vs skipped i)\n - user-cluster item CTR (kmeans on user SVD vectors, 64 & 256 clusters), smoothed toward item CTR\n - within-event z-score and rank for each"""\nimport sys, os, time, numpy as np, polars as pl\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\nfrom sklearn.cluster import MiniBatchKMeans\nsys.path.insert(0, \'@@WORK@@/v3\')\nfrom features import load, click_log\nfrom split import J20, DAY, first_events\nOUT = os.environ.get(\'X_OUT\', \'@@WORK@@/feat_x\'); os.makedirs(OUT, exist_ok=True)\nTE, TR, EE, ER, NI, NU = load(\'@@WORK@@/v3/data\')\nSPEC = {**{f\'f{k}\': (J20 + k * DAY, DAY) for k in range(1, 7)}, \'val25\': (J20 + 5 * DAY, None), \'val26\': (J20 + 6 * DAY, None), \'eval\': (J20 + 7 * DAY, \'eval\'),\n **{f\'e{k}\': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f\'E{k}\': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f\'d{k}\': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}}\ndef nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)\ndef csr(u, i, w): return sparse.csr_matrix((w.astype(np.float32), (u, i)), shape=(NU, NI))\ndef build(C, T_ev, T_rows):\n P_ev = TE.filter(pl.col(\'ts\') < C)\n P_rows = TR.join(P_ev.select(\'eid\', \'u\', \'ts\'), on=\'eid\', how=\'inner\')\n CL = click_log(C, P_ev, P_rows, T_ev) # user-level: train data + each target row\'s own history\n CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # global/item-level: train data only\n assert CL[\'ts\'].max() < C and P_rows[\'ts\'].max() < C\n R = T_rows.join(T_ev.select(\'eid\', \'u\'), on=\'eid\', how=\'inner\').select(\'rowid\', \'eid\', \'u\', \'ii\')\n uu = R[\'u\'].to_numpy().astype(np.int64); ii = R[\'ii\'].to_numpy().astype(np.int64)\n cg = CLg.group_by(\'u\', \'ii\').agg(pl.len().alias(\'n\'))\n Xc = csr(cg[\'u\'].to_numpy(), cg[\'ii\'].to_numpy(), np.ones(cg.height)); Scg = Xc.copy()\n cnt = CL.group_by(\'u\', \'ii\').agg(pl.len().alias(\'n\'), pl.col(\'ts\').max().alias(\'lt\'))\n cu = cnt[\'u\'].to_numpy(); ci = cnt[\'ii\'].to_numpy()\n df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)\n Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc\n Xu = csr(cu, ci, np.ones(cnt.height)) @ sparse.diags(idf); Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu\n neg = P_rows.filter(pl.col(\'clicked\') == 0)\n nu_ = neg[\'u\'].to_numpy(); ni_ = neg[\'ii\'].to_numpy()\n Sc = csr(cu, ci, np.ones(cnt.height)); Sn = csr(nu_, ni_, np.ones(neg.height))\n rc = np.exp(-(C - cnt[\'lt\'].to_numpy()) / (2 * DAY)); rn = np.exp(-(C - neg[\'ts\'].to_numpy()) / (2 * DAY))\n Scr = csr(cu, ci, rc); Snr = csr(nu_, ni_, rn)\n F = {}\n for k in (32, 64, 128):\n U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)\n Vn = nrm(Vt.T * np.sqrt(S)); Un = nrm(np.asarray(Xu @ (Vt.T / np.sqrt(S)))); cand = Vn[ii] # user fold-in\n nC = nrm(np.asarray(Sc @ Vn)); nN = nrm(np.asarray(Sn @ Vn))\n F[f\'x_con05_k{k}\'] = (nrm(nC - 0.5 * nN)[uu] * cand).sum(1)\n if k == 64:\n nCr = nrm(np.asarray(Scr @ Vn)); nNr = nrm(np.asarray(Snr @ Vn))\n for a, nm in ((0.5, \'05\'), (1.0, \'1\')):\n F[f\'x_conr{nm}_k64\'] = (nrm(nCr - a * nNr)[uu] * cand).sum(1)\n F[\'x_crcos_k64\'] = (nCr[uu] * cand).sum(1)\n # item side: centroid of users who clicked item minus users who skipped it, vs target user vector\n # item side / clusters: user vectors from TRAIN-ONLY clicks (Ug), so no target (eval) histories enter item-level stats\n Ug = nrm(np.asarray(Xc @ (Vt.T / np.sqrt(S))))\n iC = nrm(np.asarray(Scg.T @ Ug)); iN = nrm(np.asarray(Sn.T @ Ug))\n F[\'x_icon05_k64\'] = (nrm(iC - 0.5 * iN)[ii] * Un[uu]).sum(1)\n F[\'x_icos_k64\'] = (iC[ii] * Un[uu]).sum(1)\n # user-cluster CTR\n act = np.unique(np.concatenate([P_rows[\'u\'].to_numpy(), cg[\'u\'].to_numpy()]))\n for nc in (64, 256):\n km = MiniBatchKMeans(nc, random_state=0, batch_size=4096, n_init=3).fit(Ug[act])\n lab = km.predict(Ug).astype(np.int32) # train users\' clusters (train-only vectors) -> cluster CTR tables\n labt = km.predict(Un[uu]).astype(np.int32) # target row\'s cluster from its own user vector (own history allowed)\n g = float(P_rows[\'clicked\'].mean())\n pr = P_rows.select(\'ii\', \'clicked\', pl.Series(\'cl\', lab[P_rows[\'u\'].to_numpy()]))\n ict = pr.group_by(\'ii\').agg(((pl.col(\'clicked\').sum() + 20 * g) / (pl.len() + 20)).alias(\'ictr\'))\n cct = pr.group_by(\'cl\', \'ii\').agg(pl.col(\'clicked\').sum().alias(\'c\'), pl.len().alias(\'n\'))\n q = pl.DataFrame({\'cl\': labt, \'ii\': ii.astype(np.int32), \'r\': np.arange(len(uu))}).join(cct, on=[\'cl\', \'ii\'], how=\'left\') \\\n .join(ict, on=\'ii\', how=\'left\').sort(\'r\').with_columns(pl.col(\'ictr\').fill_null(g), pl.col(\'c\').fill_null(0), pl.col(\'n\').fill_null(0))\n F[f\'x_clctr{nc}\'] = ((q[\'c\'] + 10 * q[\'ictr\']) / (q[\'n\'] + 10)).to_numpy()\n F[f\'x_clctr{nc}_lift\'] = F[f\'x_clctr{nc}\'] / q[\'ictr\'].to_numpy()\n F[f\'x_climp{nc}\'] = q[\'n\'].to_numpy().astype(np.float32)\n R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])\n zc = [c for c in F if not c.startswith(\'x_climp\')]\n R = R.with_columns([((pl.col(c) - pl.col(c).mean().over(\'eid\')) / (pl.col(c).std().over(\'eid\') + 1e-6)).alias(c + \'_z\') for c in zc] +\n [pl.col(c).rank(\'average\').over(\'eid\').alias(c + \'_rk\') for c in zc])\n return R.drop(\'eid\', \'u\', \'ii\')\nfor name in sys.argv[1:]:\n t0 = time.time(); C, W = SPEC[name]\n if W == \'eval\': R = build(C, EE, ER)\n else:\n T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select(\'eid\'), on=\'eid\', how=\'semi\'))\n R.write_parquet(f\'{OUT}/{name}.parquet\'); print(name, R.shape, f\'{time.time()-t0:.0f}s\', flush=True)\n', 'feats.py': '"""Feature builder for TechArena CTR. All stats for a cutoff C use only events with ts < C\n(plus the target events\' own histories, which by construction contain only clicks < their ts)."""\nimport polars as pl, numpy as np, time\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\n\nDAY = 86400\nK_SMOOTH = 20\n\ndef log(*a):\n print(time.strftime(\'%H:%M:%S\'), *a, flush=True)\n\ndef load(D):\n TE = pl.read_parquet(f\'{D}/train_events.parquet\')\n TR = pl.read_parquet(f\'{D}/train_rows.parquet\')\n EE = pl.read_parquet(f\'{D}/evaluation_events.parquet\')\n ER = pl.read_parquet(f\'{D}/evaluation_rows.parquet\')\n # global item index\n items = pl.concat([TR[\'item\'], ER[\'item\'], TE[\'h_items\'].explode().drop_nulls(), EE[\'h_items\'].explode().drop_nulls()]).unique().sort()\n imap = pl.DataFrame({\'item\': items, \'ii\': pl.arange(0, len(items), eager=True).cast(pl.Int32)})\n def mapi(df, col=\'item\'):\n return df.join(imap.rename({\'item\': col, \'ii\': col + \'_ii\'}), on=col, how=\'left\')\n TR = mapi(TR).drop(\'item\').rename({\'item_ii\': \'ii\'})\n ER = mapi(ER).drop(\'item\').rename({\'item_ii\': \'ii\'})\n def map_hist(E):\n ex = E.select(\'eid\', \'h_items\').explode(\'h_items\').with_row_index(\'r\')\n ex = ex.join(imap.rename({\'item\': \'h_items\'}), on=\'h_items\', how=\'left\').sort(\'r\')\n h = ex.group_by(\'eid\', maintain_order=True).agg(pl.col(\'ii\').drop_nulls().alias(\'h\'))\n return E.drop(\'h_items\').join(h, on=\'eid\', how=\'left\').with_columns(pl.col(\'h\').fill_null([]))\n TE = map_hist(TE); EE = map_hist(EE)\n return TE, TR, EE, ER, len(items)\n\ndef first_events(TE, C, W):\n """first event per user in [C, C+W)"""\n return (TE.filter((pl.col(\'ts\') >= C) & (pl.col(\'ts\') < C + W)).sort(\'ts\')\n .group_by(\'user\', maintain_order=True).first())\n\ndef build(C, T_ev, T_rows, TE, TR, NI, svd_k=64, with_label=True):\n t0 = time.time()\n P_ev = TE.filter(pl.col(\'ts\') < C)\n P_rows = TR.join(P_ev.select(\'eid\', \'user\', \'ts\', \'tb\'), on=\'eid\', how=\'inner\')\n g = P_rows[\'clicked\'].mean() if len(P_rows) else 0.063\n ndays = max((C - P_ev[\'ts\'].min()) / DAY, 1e-3) if len(P_ev) else 1e-3\n\n # ---------- click pool (user, item, ts) ----------\n def hist_clicks(E):\n return (E.select(\'user\', \'h\', \'h_ts\').explode([\'h\', \'h_ts\']).drop_nulls()\n .rename({\'h\': \'ii\', \'h_ts\': \'ts\'}))\n pool = pl.concat([hist_clicks(P_ev), hist_clicks(T_ev),\n P_rows.filter(pl.col(\'clicked\') == 1).select(\'user\', \'ii\', \'ts\')]).unique()\n pool = pool.with_columns(pl.col(\'ii\').cast(pl.Int32), pl.col(\'ts\').cast(pl.Int64))\n # global/item-level pool: train data only (no target-event histories, so nothing aggregates across eval rows);\n # `pool` (with each target row\'s own history) is used only for that user\'s own features\n poolg = pl.concat([hist_clicks(P_ev), P_rows.filter(pl.col(\'clicked\') == 1).select(\'user\', \'ii\', \'ts\')]).unique() \\\n .with_columns(pl.col(\'ii\').cast(pl.Int32), pl.col(\'ts\').cast(pl.Int64))\n log(\'pool\', len(pool))\n\n # ---------- item stats ----------\n it = P_rows.group_by(\'ii\').agg(pl.len().alias(\'i_imp\'), pl.col(\'clicked\').sum().alias(\'i_clk\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'i_imp1d\'),\n pl.col(\'clicked\').filter(pl.col(\'ts\') >= C - DAY).sum().alias(\'i_clk1d\'))\n it = it.with_columns(((pl.col(\'i_clk\') + K_SMOOTH * g) / (pl.col(\'i_imp\') + K_SMOOTH)).alias(\'i_ctr\'),\n ((pl.col(\'i_clk1d\') + K_SMOOTH * g) / (pl.col(\'i_imp1d\') + K_SMOOTH)).alias(\'i_ctr1d\'),\n (pl.col(\'i_imp\') / ndays).alias(\'i_imp_rate\'))\n ip = poolg.group_by(\'ii\').agg(pl.len().alias(\'ip_n\'), pl.col(\'user\').n_unique().alias(\'ip_users\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'ip_1d\'),\n (pl.col(\'ts\') >= C - 3 * DAY).sum().alias(\'ip_3d\'),\n ((pl.col(\'ts\') - C) / DAY).exp().sum().alias(\'ip_dec1\'),\n ((pl.col(\'ts\') - C) / (3 * DAY)).exp().sum().alias(\'ip_dec3\'),\n (pl.col(\'ts\') < 1737331200).sum().alias(\'ip_warm\'))\n itb = P_rows.group_by(\'ii\', \'tb\').agg(pl.len().alias(\'n\'), pl.col(\'clicked\').sum().alias(\'c\'))\n itb = itb.with_columns(((pl.col(\'c\') + K_SMOOTH * g) / (pl.col(\'n\') + K_SMOOTH)).alias(\'i_ctr_tb\')).select(\'ii\', \'tb\', \'i_ctr_tb\')\n\n # ---------- user stats ----------\n ue = P_rows.group_by(\'user\').agg(pl.col(\'eid\').n_unique().alias(\'u_ev\'), pl.col(\'clicked\').sum().alias(\'u_clk\'),\n pl.col(\'ts\').max().alias(\'u_last_ev\'))\n ue = ue.with_columns(((pl.col(\'u_clk\') + 5 * g * 10) / (pl.col(\'u_ev\') + 5)).alias(\'u_ctr\'))\n up = pool.group_by(\'user\').agg(pl.len().alias(\'up_n\'), pl.col(\'ts\').max().alias(\'up_last\'),\n pl.col(\'ii\').n_unique().alias(\'up_uniq\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'up_1d\'))\n # user-item prior impressions\n ui = P_rows.group_by(\'user\', \'ii\').agg(pl.len().alias(\'ui_imp\'), pl.col(\'clicked\').sum().alias(\'ui_clk\'),\n pl.col(\'ts\').max().alias(\'ui_last_imp\'))\n uip = pool.group_by(\'user\', \'ii\').agg(pl.len().alias(\'uip_n\'), pl.col(\'ts\').max().alias(\'uip_last\'))\n\n # ---------- co-occurrence (user sets) & transitions ----------\n us = poolg.select(\'user\', \'ii\').unique()\n # v8: user-set co-occurrence counts via sparse B^T B (same integer counts as the former self-join, far less memory)\n _ur = us[\'user\'].rank(\'dense\').to_numpy().astype(np.int64) - 1\n _B = sparse.csr_matrix((np.ones(len(us), np.int32), (_ur, us[\'ii\'].to_numpy().astype(np.int64))), shape=(int(_ur.max()) + 1 if len(us) else 0, NI))\n _C = (_B.T @ _B).tocsr(); _pop = _C.diagonal().astype(np.float64); _C.setdiag(0); _C.eliminate_zeros()\n del _B, _ur\n log(\'cooc pairs\', _C.nnz) # _C / _pop replace the (ii, ii_b, co, cos) frame + hash join\n seq = poolg.sort(\'user\', \'ts\')\n tr_pairs = []\n for lag in (1, 2, 3):\n tr_pairs.append(seq.with_columns(pl.col(\'ii\').shift(lag).over(\'user\').alias(\'prev\'))\n .drop_nulls(\'prev\').select(pl.col(\'prev\').cast(pl.Int32), \'ii\', pl.lit(1.0 / lag).alias(\'w\')))\n trn = pl.concat(tr_pairs).group_by(\'prev\', \'ii\').agg(pl.col(\'w\').sum().alias(\'trn\'))\n trn_out = trn.group_by(\'prev\').agg(pl.col(\'trn\').sum().alias(\'trn_tot\'))\n trn = trn.join(trn_out, on=\'prev\').with_columns((pl.col(\'trn\') / (pl.col(\'trn_tot\') + 5)).cast(pl.Float32).alias(\'trp\')).select(\'prev\', \'ii\', \'trn\', \'trp\')\n\n # ---------- SVD embeddings of user x item click matrix ----------\n uidx = us.select(\'user\').unique().with_row_index(\'ur\')\n m = us.join(uidx, on=\'user\')\n df_ = np.bincount(m[\'ii\'].to_numpy(), minlength=NI)\n idf = np.log((len(uidx) + 1) / (df_ + 1)).astype(np.float32)\n X = sparse.csr_matrix((idf[m[\'ii\'].to_numpy()], (m[\'ur\'].to_numpy(), m[\'ii\'].to_numpy())), shape=(len(uidx), NI))\n rn = np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel()) + 1e-6\n X = sparse.diags(1 / rn) @ X\n U, S, Vt = randomized_svd(X, svd_k, random_state=0, n_iter=4)\n V = Vt.T.astype(np.float32) # unscaled singular vectors (best in emb_exp)\n V /= (np.linalg.norm(V, axis=1, keepdims=True) + 1e-6)\n V32 = V[:, :32] / (np.linalg.norm(V[:, :32], axis=1, keepdims=True) + 1e-6)\n # fold-in of every target user\'s own pool (train clicks + own history) onto the train-only factors\n uidx = pool.select(\'user\').unique().with_row_index(\'ur\')\n mu = pool.select(\'user\', \'ii\').unique().join(uidx, on=\'user\')\n Xu = sparse.csr_matrix((idf[mu[\'ii\'].to_numpy()], (mu[\'ur\'].to_numpy(), mu[\'ii\'].to_numpy())), shape=(len(uidx), NI))\n Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu\n U = np.asarray(Xu @ (Vt.T / S)).astype(np.float32)\n S = np.ones_like(S)\n log(\'svd done\')\n\n # ---------- assemble target rows ----------\n R = T_rows.join(T_ev, on=\'eid\', how=\'left\').with_row_index(\'r\')\n R = R.with_columns((pl.col(\'ts\') - C).alias(\'stale\'), ((pl.col(\'ts\') // 3600) % 24).cast(pl.Int8).alias(\'hour\'))\n R = (R.join(it, on=\'ii\', how=\'left\').join(ip, on=\'ii\', how=\'left\').join(itb, on=[\'ii\', \'tb\'], how=\'left\')\n .join(ue, on=\'user\', how=\'left\').join(up, on=\'user\', how=\'left\')\n .join(ui, on=[\'user\', \'ii\'], how=\'left\').join(uip, on=[\'user\', \'ii\'], how=\'left\').sort(\'r\'))\n R = R.with_columns(\n pl.col(\'h\').list.len().alias(\'h_len\'),\n (pl.col(\'ts\') - pl.col(\'h_ts\').list.last()).alias(\'h_since_last\'),\n (pl.col(\'ts\') - pl.col(\'h_ts\').list.first()).alias(\'h_since_first\'),\n pl.col(\'h\').list.n_unique().alias(\'h_uniq\'),\n pl.col(\'h_ts\').list.eval((pl.element() >= pl.element().max() - DAY).sum()).list.first().alias(\'h_1d_before_last\'),\n (pl.col(\'ts\') - pl.col(\'u_last_ev\')).alias(\'u_since_ev\'),\n (pl.col(\'ts\') - pl.col(\'up_last\')).alias(\'up_since\'),\n (pl.col(\'ts\') - pl.col(\'ui_last_imp\')).alias(\'ui_since_imp\'),\n (pl.col(\'ts\') - pl.col(\'uip_last\')).alias(\'uip_since\'),\n (pl.col(\'ui_imp\') - pl.col(\'ui_clk\')).alias(\'ui_noclk\'),\n (pl.col(\'ip_n\') / (pl.col(\'i_imp\') + 1)).alias(\'ip_per_imp\'),\n (pl.col(\'i_ctr_tb\') - pl.col(\'i_ctr\')).alias(\'i_ctr_tb_diff\'),\n )\n R = R.with_columns((pl.col(\'h_since_first\') / (pl.col(\'h_len\') - 1).clip(1)).alias(\'h_gap_mean\'),\n (pl.col(\'h_uniq\') / pl.col(\'h_len\').clip(1)).alias(\'h_uniq_ratio\'))\n\n # pairwise candidate x history\n PR = (R.select(\'r\', \'ii\', \'h\', \'h_ts\', \'ts\').with_columns(pl.int_ranges(pl.col(\'h\').list.len(), 0, -1).alias(\'pos\'))\n .explode([\'h\', \'h_ts\', \'pos\']).drop_nulls(\'h\')\n .with_columns(pl.col(\'h\').cast(pl.Int32)))\n _pi = PR[\'ii\'].to_numpy().astype(np.int64); _ph = PR[\'h\'].to_numpy().astype(np.int64)\n _cv = np.asarray(_C[_pi, _ph]).ravel().astype(np.float64)\n _cs = np.where(_cv > 0, _cv / np.sqrt(np.maximum(_pop[_pi] * _pop[_ph], 1e-12)), 0.0).astype(np.float32)\n PR = PR.with_columns(pl.Series(\'co\', _cv.astype(np.int32)), pl.Series(\'cos\', _cs)); del _pi, _ph, _cv, _cs\n PR = PR.with_columns((0.85 ** (pl.col(\'pos\') - 1)).alias(\'rw\'), (pl.col(\'h\') == pl.col(\'ii\')).alias(\'same\'))\n agg = PR.group_by(\'r\').agg(\n pl.col(\'cos\').sum().alias(\'co_sum\'), pl.col(\'cos\').max().alias(\'co_max\'), pl.col(\'cos\').mean().alias(\'co_mean\'),\n (pl.col(\'cos\') * pl.col(\'rw\')).sum().alias(\'co_rw\'), pl.col(\'co\').sum().alias(\'co_cnt\'),\n pl.col(\'cos\').filter(pl.col(\'pos\') <= 3).sum().alias(\'co_last3\'),\n pl.col(\'same\').sum().alias(\'inh_cnt\'), pl.col(\'pos\').filter(pl.col(\'same\')).min().alias(\'inh_pos\'),\n (pl.col(\'ts\') - pl.col(\'h_ts\')).filter(pl.col(\'same\')).min().alias(\'inh_since\'),\n (pl.col(\'cos\') > 0).sum().alias(\'co_nz\'))\n R = R.join(agg, on=\'r\', how=\'left\')\n # transitions from last 3 history items\n TRp = (R.select(\'r\', \'ii\', pl.col(\'h\').list.tail(3).alias(\'h3\')).with_columns(pl.int_ranges(pl.col(\'h3\').list.len(), 0, -1).alias(\'pos\'))\n .explode([\'h3\', \'pos\']).drop_nulls(\'h3\').rename({\'h3\': \'prev\'}).with_columns(pl.col(\'prev\').cast(pl.Int32))\n .join(trn, on=[\'prev\', \'ii\'], how=\'left\').with_columns(pl.col(\'trn\').fill_null(0), pl.col(\'trp\').fill_null(0)))\n tagg = TRp.group_by(\'r\').agg(pl.col(\'trn\').filter(pl.col(\'pos\') == 1).sum().alias(\'tr_last\'),\n pl.col(\'trp\').filter(pl.col(\'pos\') == 1).sum().alias(\'trp_last\'),\n pl.col(\'trn\').sum().alias(\'tr_last3\'), pl.col(\'trp\').sum().alias(\'trp_last3\'))\n R = R.join(tagg, on=\'r\', how=\'left\').sort(\'r\')\n # SVD cosine: recency weighted mean of history vectors\n ex = PR.select(\'r\', \'h\', \'rw\')\n Wrh = sparse.csr_matrix((ex[\'rw\'].to_numpy().astype(np.float32), (ex[\'r\'].to_numpy(), ex[\'h\'].to_numpy())), shape=(len(R), NI)) # v8: replaces np.add.at\n hv = np.asarray(Wrh @ V, dtype=np.float32)\n hvn = hv / (np.linalg.norm(hv, axis=1, keepdims=True) + 1e-6)\n cand = V[R[\'ii\'].to_numpy()]\n svd_cos = (hvn * cand).sum(1)\n # user factor from SVD (pool-based, covers clicks beyond last 20)\n uf = np.zeros((len(R), V.shape[1]), np.float32)\n um = R.select(\'user\').join(uidx, on=\'user\', how=\'left\')[\'ur\'].to_numpy()\n ok = ~np.isnan(um.astype(float)) if um.dtype.kind == \'f\' else np.ones(len(um), bool)\n umi = np.nan_to_num(um.astype(float), nan=0).astype(np.int64)\n ufv = (U[umi] * np.sqrt(S)).astype(np.float32); ufv[~ok] = 0\n ufv /= (np.linalg.norm(ufv, axis=1, keepdims=True) + 1e-6)\n svd_ucos = (ufv * cand).sum(1)\n hv32 = np.asarray(Wrh @ V32, dtype=np.float32)\n svd32_cos = (hv32 / (np.linalg.norm(hv32, axis=1, keepdims=True) + 1e-6) * V32[R[\'ii\'].to_numpy()]).sum(1)\n extra_svd = {}\n for kk in (16, 24, 48):\n Vk = V[:, :kk] / (np.linalg.norm(V[:, :kk], axis=1, keepdims=True) + 1e-6)\n hk = np.asarray(Wrh @ Vk, dtype=np.float32)\n extra_svd[f\'svd{kk}_cos\'] = (hk / (np.linalg.norm(hk, axis=1, keepdims=True) + 1e-6) * Vk[R[\'ii\'].to_numpy()]).sum(1)\n # pairwise svd similarity: max over history, and with the most recent item\n pr_r = PR[\'r\'].to_numpy(); pr_c = PR[\'ii\'].to_numpy(); pr_h = PR[\'h\'].to_numpy()\n sims = np.empty(len(PR), np.float32)\n for k in range(0, len(PR), 2_000_000):\n sl = slice(k, k + 2_000_000); sims[sl] = (V[pr_c[sl]] * V[pr_h[sl]]).sum(1)\n sp = pl.DataFrame({\'r\': pr_r, \'sim\': sims, \'pos\': PR[\'pos\'].to_numpy(), \'same\': PR[\'same\'].to_numpy()}).filter(~pl.col(\'same\'))\n sagg = sp.group_by(\'r\').agg(pl.col(\'sim\').max().alias(\'svd_maxsim\'), pl.col(\'sim\').filter(pl.col(\'pos\') == 1).max().alias(\'svd_last\'),\n pl.col(\'sim\').top_k(3).mean().alias(\'svd_top3\'))\n R = R.with_columns(pl.Series(\'svd_cos\', svd_cos), pl.Series(\'svd_ucos\', svd_ucos), pl.Series(\'svd_hnorm\', np.linalg.norm(hv, axis=1)),\n pl.Series(\'svd32_cos\', svd32_cos), *[pl.Series(k, v) for k, v in extra_svd.items()])\n R = R.join(sagg, on=\'r\', how=\'left\').sort(\'r\')\n\n fill0 = [\'i_imp\', \'i_clk\', \'i_imp1d\', \'i_clk1d\', \'ip_n\', \'ip_users\', \'ip_1d\', \'ip_3d\', \'ip_dec1\', \'ip_dec3\', \'ip_warm\',\n \'u_ev\', \'u_clk\', \'up_n\', \'up_uniq\', \'up_1d\', \'ui_imp\', \'ui_clk\', \'ui_noclk\', \'uip_n\', \'co_sum\', \'co_max\', \'co_mean\', \'co_rw\',\n \'co_cnt\', \'co_last3\', \'inh_cnt\', \'co_nz\', \'tr_last\', \'trp_last\', \'tr_last3\', \'trp_last3\']\n R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])\n R = R.with_columns(pl.col(\'i_ctr\').fill_null(g), pl.col(\'i_ctr1d\').fill_null(g), pl.col(\'i_ctr_tb\').fill_null(g), pl.col(\'u_ctr\').fill_null(g * 10))\n # within-event relative features\n rel = [\'i_ctr\', \'ip_dec1\', \'ip_n\', \'co_sum\', \'co_rw\', \'co_max\', \'svd_cos\', \'svd_ucos\', \'svd32_cos\', \'svd16_cos\', \'svd24_cos\', \'svd48_cos\', \'svd_maxsim\', \'svd_top3\', \'tr_last3\', \'uip_n\', \'i_ctr_tb\', \'ui_noclk\']\n R = R.with_columns([pl.col(c).rank(\'average\').over(\'eid\').alias(c + \'_rk\') for c in rel] +\n [(pl.col(c) - pl.col(c).mean().over(\'eid\')).alias(c + \'_dm\') for c in rel] +\n [(pl.col(c) / (pl.col(c).max().over(\'eid\') + 1e-9)).alias(c + \'_rmax\') for c in [\'co_sum\', \'svd_cos\', \'ip_dec1\', \'co_rw\']])\n R = R.with_columns(pl.col(\'co_rw\').max().over(\'eid\').alias(\'ev_co_max\'), pl.col(\'svd_cos\').max().over(\'eid\').alias(\'ev_svd_max\'),\n pl.col(\'svd_cos\').mean().over(\'eid\').alias(\'ev_svd_mean\'),\n (pl.col(\'inh_cnt\') > 0).sum().over(\'eid\').alias(\'ev_n_inh\'), pl.col(\'i_ctr\').mean().over(\'eid\').alias(\'ev_ictr_mean\'),\n pl.col(\'ip_dec1\').mean().over(\'eid\').alias(\'ev_ipdec_mean\'))\n log(f\'build C={C} rows={len(R)} {time.time()-t0:.0f}s\')\n starter = dict(g=g)\n return R\n\nDROP = [\'r\', \'rowid\', \'eid\', \'user\', \'ii\', \'h\', \'h_ts\', \'ts\', \'clicked\', \'u_last_ev\', \'up_last\', \'ui_last_imp\', \'uip_last\', \'fold\']\n', 'lrfeat2.py': '"""F1 (data_dig_v1): history-centroid cosine in an exact low-rank SVD space. Train-only, rebuilt at each fold cutoff C.\nMatrix: users x items, binary, rows = each user\'s last N distinct history clicks (from train events with ts < C),\ncolumns scaled by 1/sqrt(pop+5); exact svds (ARPACK) k=17; item vectors = comps 2..K+1 (popularity comp dropped) * s.\nFeature: cos(candidate vector, sum of own-history normalised vectors). Within-event _z and _dmax. Eval: fitted at J27 on train only.\nUsage: python lrfeat.py NAME... -> feat_lr/NAME.parquet (rowid, lr{K}_cos/_z/_dmax for K in 10,12,16)"""\nimport sys, os, time, numpy as np, polars as pl, scipy.sparse as sp\nfrom scipy.sparse.linalg import svds\nsys.path.insert(0, \'@@WORK@@/v3\')\nfrom features import load\nfrom split import J20, DAY, first_events\nTE, TR, EE, ER, NI, NU = load(\'@@WORK@@/v3/data\')\nN = int(os.environ.get(\'LR_N\', 40)); OUT = os.environ.get(\'LR_OUT\', \'@@WORK@@/feat_lr2\'); os.makedirs(OUT, exist_ok=True)\nKS = tuple(int(x) for x in os.environ.get(\'LR_KS\', \'10,12,16\').split(\',\'))\nCON = int(os.environ.get(\'LR_CON\', 0))\nSPEC = {**{f\'f{k}\': (J20 + k * DAY, DAY) for k in range(1, 7)}, \'val25\': (J20 + 5 * DAY, None), \'val26\': (J20 + 6 * DAY, None)}\n_V = {}\ndef vecs(C):\n if C in _V: return _V[C]\n h = (TE.filter(pl.col(\'ts\') < C).select(\'u\', \'h\', \'h_ts\').explode([\'h\', \'h_ts\']).drop_nulls(\'h\')\n .select(\'u\', pl.col(\'h\').cast(pl.Int32).alias(\'ii\'), pl.col(\'h_ts\').cast(pl.Int64).alias(\'t\')).unique().sort(\'u\', \'t\', \'ii\'))\n h = h.with_columns(pl.int_range(pl.len()).over(\'u\').alias(\'p\'), pl.len().over(\'u\').alias(\'L\')).filter(pl.col(\'L\') - pl.col(\'p\') <= N).unique([\'u\', \'ii\'])\n M = sp.csr_matrix((np.ones(h.height, np.float32), (h[\'u\'].to_numpy(), h[\'ii\'].to_numpy())), shape=(NU, NI))\n pop = np.asarray(M.sum(0)).ravel(); M = M.multiply(1 / np.sqrt(pop + 5)).tocsr()\n _, s, vt = svds(M, k=max(max(KS) + 1, 17), random_state=0); o = np.argsort(-s); s, vt = s[o], vt[o]\n print(time.strftime(\'%T\'), \'C\', C, \'pairs\', h.height, \'sv\', np.round(s[:15], 2), flush=True)\n _V[C] = (s, vt); return _V[C]\ndef build(C, T_ev, T_rows):\n s, vt = vecs(C)\n E = T_ev.select(\'eid\', \'h\').with_row_index(\'k\'); kk = E.select(\'eid\', \'k\')\n H = E.select(\'k\', \'h\').explode(\'h\').drop_nulls(\'h\'); hk = H[\'k\'].to_numpy(); hi = H[\'h\'].to_numpy().astype(np.int64)\n R = T_rows.select(\'rowid\', \'eid\', \'ii\').join(kk, on=\'eid\', how=\'left\'); rk = R[\'k\'].to_numpy(); ri = R[\'ii\'].to_numpy().astype(np.int64)\n F = {}\n if CON:\n P_ev = TE.filter(pl.col(\'ts\') < C).select(\'eid\', \'u\')\n ng = TR.filter(pl.col(\'clicked\') == 0).join(P_ev, on=\'eid\', how=\'inner\').select(\'u\', \'ii\').unique()\n ue = T_ev.select(\'u\').with_row_index(\'k\').join(ng, on=\'u\', how=\'inner\'); nk = ue[\'k\'].to_numpy(); ni = ue[\'ii\'].to_numpy().astype(np.int64)\n Shk = sp.csr_matrix((np.ones(len(hk)), (hk, hi)), shape=(E.height, NI))\n if CON: Snk = sp.csr_matrix((np.ones(len(nk)), (nk, ni)), shape=(E.height, NI))\n for K in KS:\n V = (vt[1:K + 1].T * s[1:K + 1]); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9)\n hc = np.asarray(Shk @ V) # v8: sparse matmul replaces np.add.at\n hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1)\n x = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan)\n F[f\'lr{K}_cos\'] = x.astype(np.float32)\n if CON:\n nc = np.asarray(Snk @ V)\n hn = hc / (np.linalg.norm(hc, axis=1, keepdims=True) + 1e-9); nn = nc / (np.linalg.norm(nc, axis=1, keepdims=True) + 1e-9)\n for al, nm in ((0.5, \'05\'), (1.0, \'1\')):\n d = (hn - al * nn)[rk]; dn = np.linalg.norm(d, axis=1)\n F[f\'lr{K}_con{nm}\'] = np.where(dn > 0, (V[ri] * d).sum(1) / (dn + 1e-9), np.nan).astype(np.float32)\n R = R.select(\'rowid\', \'eid\').with_columns([pl.Series(c, v).fill_nan(None) for c, v in F.items()])\n R = R.with_columns([((pl.col(c) - pl.col(c).mean().over(\'eid\')) / (pl.col(c).std().over(\'eid\') + 1e-6)).alias(c.replace(\'_cos\', \'_z\') if c.endswith(\'_cos\') else c + \'_z\') for c in F] +\n [(pl.col(c) - pl.col(c).max().over(\'eid\')).alias(c.replace(\'_cos\', \'_dmax\') if c.endswith(\'_cos\') else c + \'_dmax\') for c in F])\n return R.drop(\'eid\')\nfor n in sys.argv[1:]:\n t0 = time.time()\n if n == \'eval\': R = build(J20 + 7 * DAY, EE, ER)\n else:\n C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select(\'eid\'), on=\'eid\', how=\'semi\'))\n R.write_parquet(f\'{OUT}/{n}.parquet\'); print(n, R.shape, f\'{time.time()-t0:.0f}s\', flush=True)\n', 'mlp12.py': '"""mlp12.py CUT TAG [--ep 4] [--seed 0]: MLP on the same LightGBM matrix (train f(C-24)..f(C-21), negfrac .5, v10B drop), rank-gauss inputs\nfrom TRAIN quantiles only + missing flags. Saves ens/{TAG}_c{CUT}_s{seed}.npy (n,EP) logits after each epoch."""\nimport os, sys, re, time, argparse, json, gc, numpy as np, torch, torch.nn as nn\nfrom cl import matrix\nfrom sklearn.metrics import roc_auc_score\nap=argparse.ArgumentParser(); ap.add_argument(\'cut\',type=int); ap.add_argument(\'tag\'); ap.add_argument(\'--ep\',type=int,default=4); ap.add_argument(\'--seed\',type=int,default=0)\nap.add_argument(\'--lr\',default=\'feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw\'); ap.add_argument(\'--hid\',type=int,default=512); ap.add_argument(\'--drop\',type=float,default=0.2)\na=ap.parse_args(); t0=time.time(); torch.manual_seed(a.seed); np.random.seed(a.seed); torch.set_num_threads(int(os.environ.get(\'MLP_THREADS\',8)))\nDR=\'^sq_(pdot|pcos|mdot|mcos)$\'; extra=tuple(a.lr.split(\',\'))\ntrn=[f\'f{k}\' for k in range(max(1,a.cut-24),a.cut-20)]\nrs=np.random.RandomState(123); Xs=[];ys=[]\nfor n in trn:\n X,m,f=matrix(n,1.0,extra=extra); y=m[\'clicked\'].to_numpy(); sel=(y==1)|(rs.rand(len(y))<0.5); Xs.append(X[sel]); ys.append(y[sel]); feats=f; del X; gc.collect()\nkeep=[i for i,c in enumerate(feats) if not re.search(DR,c)]; fl=[feats[i] for i in keep]\nX=np.concatenate([x[:,keep] for x in Xs]); del Xs; y=np.concatenate(ys).astype(np.float32); gc.collect()\nvn=\'eval\' if a.cut==27 else f\'val{a.cut}\'; Xv,mv,fv=matrix(vn,1.0,extra=extra); Xv=Xv[:,[fv.index(c) for c in fl]]\n# rank-gauss via train quantiles (101 knots); NaN -> 0 + flag\nfrom scipy.special import ndtri\nqs=np.linspace(0.001,0.999,101); Q=np.nanquantile(X[::7],qs,axis=0).astype(np.float32) # (101,F)\nnanc=np.where(np.isnan(X[::7]).any(0)|np.isnan(Xv[::7]).any(0))[0]; G=ndtri(qs).astype(np.float32)\ndef tr(Z):\n out=np.empty((len(Z),Z.shape[1]+len(nanc)),np.float32)\n for j in range(Z.shape[1]):\n q=np.nan_to_num(Q[:,j],nan=0.0); v=Z[:,j]; u,ix=np.unique(q,return_index=True)\n o=np.interp(v,u,G[ix]) if len(u)>1 else np.zeros_like(v); o[np.isnan(v)]=0; out[:,j]=o\n out[:,Z.shape[1]:]=np.isnan(Z[:,nanc]); return out\nX=np.nan_to_num(tr(X)); Xv=np.nan_to_num(tr(Xv)); gc.collect(); print(\'prep\',X.shape,round(time.time()-t0),\'s\',flush=True)\nD=X.shape[1]\nnet=nn.Sequential(nn.Linear(D,a.hid),nn.BatchNorm1d(a.hid),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid,a.hid//2),nn.BatchNorm1d(a.hid//2),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid//2,1))\nopt=torch.optim.AdamW(net.parameters(),1e-3,weight_decay=1e-4); lf=nn.BCEWithLogitsLoss(); Xt=torch.from_numpy(X); Yt=torch.from_numpy(y); Xvt=torch.from_numpy(Xv)\nbs=2048; steps=a.ep*((len(y)+bs-1)//bs); sch=torch.optim.lr_scheduler.OneCycleLR(opt,2e-3,total_steps=steps,pct_start=0.1); S=[]\nfor ep in range(a.ep):\n net.train(); perm=torch.randperm(len(y))\n for b in range(0,len(y),bs):\n j=perm[b:b+bs]; opt.zero_grad(); l=lf(net(Xt[j]).squeeze(-1),Yt[j]); l.backward(); opt.step(); sch.step()\n net.eval()\n with torch.no_grad(): p=torch.cat([net(Xvt[b:b+65536]).squeeze(-1) for b in range(0,len(Xv),65536)]).numpy()\n S.append(p); msg=f\'ep{ep} {time.time()-t0:.0f}s\'\n if a.cut!=27: msg+=f\' auc {roc_auc_score(mv["clicked"].to_numpy(),p):.5f}\'\n print(msg,flush=True)\nos.makedirs(\'ens\',exist_ok=True); np.save(f\'ens/{a.tag}_c{a.cut}_s{a.seed}.npy\',np.stack(S,1).astype(np.float32))\nopen(\'ens/ens_log.jsonl\',\'a\').write(json.dumps(dict(cut=a.cut,tag=a.tag,seed=a.seed,min=round((time.time()-t0)/60,1)))+\'\\n\')\n', 'ppmi.py': '"""v8 (ported from v5 ppmi.py): sequence-window PPMI-SVD item embeddings from CLg only (train clicks + train-event histories, ts WORK/feat_pp/NAME.parquet"""\nimport sys, os, time, json, numpy as np, polars as pl\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\nsys.path.insert(0, \'@@WORK@@/v3\')\nfrom features import load, click_log\nfrom split import J20, DAY, first_events\nOUT = \'@@WORK@@/feat_pp\'; os.makedirs(OUT, exist_ok=True)\nTE, TR, EE, ER, NI, NU = load(\'@@WORK@@/v3/data\')\nSPEC = {**{f\'f{k}\': (J20 + k * DAY, DAY) for k in range(1, 7)}, \'val23\': (J20 + 3 * DAY, None), \'val25\': (J20 + 5 * DAY, None), \'val26\': (J20 + 6 * DAY, None), \'eval\': (J20 + 7 * DAY, \'eval\')}\ndef nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)\ndef csr(u, i, w, shape): return sparse.csr_matrix((np.asarray(w, np.float32), (u, i)), shape=shape)\ndef ppmi_emb(CLg, ks=(32, 64)):\n s = CLg.select(\'u\', pl.col(\'ii\').cast(pl.Int64)).to_numpy() # sorted by u, ts\n u, it = s[:, 0], s[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32)\n for d in range(1, 6):\n m = u[d:] == u[:-d]; a, b = it[:-d][m], it[d:][m]; ok = a != b; a, b = a[ok], b[ok]\n A = A + csr(a, b, np.full(len(a), 1.0 / d), (NI, NI)) + csr(b, a, np.full(len(a), 1.0 / d), (NI, NI))\n A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum()\n pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0\n M = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI))\n E = {}\n for k in ks:\n U, S, Vt = randomized_svd(M, k, random_state=0, n_iter=5); E[k] = nrm(U * np.sqrt(S))\n return E, int(len(a))\ndef build(C, T_ev, T_rows):\n P_ev = TE.filter(pl.col(\'ts\') < C); P_rows = TR.join(P_ev.select(\'eid\', \'u\', \'ts\'), on=\'eid\', how=\'inner\')\n CL = click_log(C, P_ev, P_rows, T_ev); CLg = click_log(C, P_ev, P_rows, T_ev.head(0))\n assert CLg[\'ts\'].max() < C and CL[\'ts\'].max() < C\n # acceptance: window counts from CLg only -> target events\' histories must not change CLg\n assert CLg.height == click_log(C, P_ev, P_rows, T_ev.head(0)).height\n E, _ = ppmi_emb(CLg)\n R = T_rows.join(T_ev.select(\'eid\', \'u\'), on=\'eid\', how=\'inner\').select(\'rowid\', \'eid\', \'u\', \'ii\')\n uu = R[\'u\'].to_numpy().astype(np.int64); ii = R[\'ii\'].to_numpy().astype(np.int64)\n cnt = CL.group_by(\'u\', \'ii\').agg(pl.len()); Sc = csr(cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy(), np.ones(cnt.height), (NU, NI))\n neg = P_rows.filter(pl.col(\'clicked\') == 0); Sn = csr(neg[\'u\'].to_numpy(), neg[\'ii\'].to_numpy(), np.ones(neg.height), (NU, NI))\n F = {}\n E32, E64 = E[32], E[64]\n nC = nrm(np.asarray(Sc @ E32)); nN = nrm(np.asarray(Sn @ E32)); F[\'x_pp_con05_k32\'] = (nrm(nC - 0.5 * nN)[uu] * E32[ii]).sum(1)\n hC = nrm(np.asarray(Sc @ E64)); F[\'x_pp_hcos\'] = (hC[uu] * E64[ii]).sum(1)\n l3 = CL.sort(\'u\', \'ts\').group_by(\'u\', maintain_order=True).tail(3).select(\'u\', pl.col(\'ii\').cast(pl.Int64))\n L = np.full((NU, 3), -1, np.int64); pos = l3.with_columns(pl.int_range(pl.len()).over(\'u\').alias(\'p\'))\n L[pos[\'u\'].to_numpy(), pos[\'p\'].to_numpy()] = pos[\'ii\'].to_numpy()\n best = np.full(len(uu), -np.inf, np.float32); Lu = L[uu]\n for j in range(3):\n v = Lu[:, j]; ok = v >= 0; sc = np.full(len(uu), -np.inf, np.float32); sc[ok] = (E64[v[ok]] * E64[ii[ok]]).sum(1)\n best = np.maximum(best, sc)\n F[\'x_pp_last3\'] = np.where(np.isfinite(best), best, 0.0)\n R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])\n R = R.with_columns([((pl.col(c) - pl.col(c).mean().over(\'eid\')) / (pl.col(c).std().over(\'eid\') + 1e-6)).alias(c + \'_z\') for c in F] +\n [pl.col(c).rank(\'average\').over(\'eid\').alias(c + \'_rk\') for c in F])\n return R\ndef table(name):\n C, W = SPEC[name]\n if W == \'eval\': return build(C, EE, ER)\n T_ev = first_events(TE, C, W); return build(C, T_ev, TR.join(T_ev.select(\'eid\'), on=\'eid\', how=\'semi\'))\nfor name in sys.argv[1:]:\n t0 = time.time(); R = table(name).drop(\'eid\', \'u\', \'ii\'); R.write_parquet(f\'{OUT}/{name}.parquet\'); print(name, R.shape, f\'{time.time()-t0:.0f}s\', flush=True)\n', 'prep.py': "import polars as pl, time, sys\nD='@@WORK@@'\ndef hexid(e, n=15): # opaque id -> int64 via first 15 hex chars after prefix\n return e.str.slice(4, n).str.to_integer(base=16)\ndef prep(name, has_label):\n t=time.time()\n lf=pl.scan_csv(f'{D}/{name}.csv', schema_overrides={'history_item_ids':pl.Utf8,'history_timestamps':pl.Utf8})\n cols=[pl.col('rowid'), hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('item_id')).alias('item')]\n if has_label: cols.append(pl.col('clicked').cast(pl.Int8))\n lf.select(cols).sink_parquet(f'{D}/{name}_rows.parquet')\n print(name,'rows done',time.time()-t,flush=True)\n ev=(lf.select('event_id','user_id','timestamp','time_bucket','day_type','history_item_ids','history_timestamps')\n .unique(subset='event_id',keep='any')\n .select(hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('user_id')).alias('user'),\n pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'),\n pl.col('day_type').cast(pl.Int8).alias('dt'),\n pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).list.eval(pl.element().str.slice(4,15).str.to_integer(base=16)).alias('h_items'),\n pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts')))\n ev.collect(engine='streaming').sort('ts').write_parquet(f'{D}/{name}_events.parquet')\n print(name,'events done',time.time()-t,flush=True)\nprep('evaluation',False)\nprep('train',True)\n", 'seq11.py': '"""v12 seq11.py (seq10.py + env knobs SQ_SEED/SQ_H/SQ_HID/SQ_DH/SQ_D/SQ_HL/SQ_TEMB; defaults == seq10). Original v10 doc: v10 seq10.py = copy of shipped v8 seq.py + extra outputs for LightGBM stacking (attention stats, user/candidate dot+cos, within-event rank/gap);\nfit() reseeds torch per cutoff so each fit is order-independent. Encoder variant via SQ_ENC (din|gru|rec).\nv8 sequence model (shipped config: last 500k train events < C, 3 epochs, batch 256, <=6000 steps, NP 3, no slate) (DIN-style target attention), refit per cutoff C on train events ts logit; BCE on all 10 slate rows.\nTarget tables only use their own history + train events < C (no aggregates over unlabelled target/eval rows).\nUsage: python seq.py NAME... [--frac 0.25] -> feat_sq/NAME.parquet (rowid, sq_logit, sq_z, sq_dm, sq_lse)"""\nimport sys, os, time, argparse, numpy as np, polars as pl, torch, torch.nn as nn\nD = os.environ.get(\'SQ_WORK\', \'@@WORK@@\')\nsys.path.insert(0, f\'{D}/v3\')\nfrom features import load, click_log\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\nfrom split import J20, DAY, first_events\nap = argparse.ArgumentParser(); ap.add_argument(\'names\', nargs=\'+\'); ap.add_argument(\'--frac\', type=float, default=1.0)\nap.add_argument(\'--ntr\', type=int, default=int(os.environ.get(\'SQ_NTR\', 500000))); ap.add_argument(\'--ep\', type=int, default=int(os.environ.get(\'SQ_EP\', 3)))\nap.add_argument(\'--d\', type=int, default=int(os.environ.get(\'SQ_D\', 32))); ap.add_argument(\'--out\', default=os.environ.get(\'SQ_OUT\', f\'{D}/feat_sq11\'))\na = ap.parse_args(); os.makedirs(a.out, exist_ok=True)\nNT = int(os.environ.get(\'SQ_THREADS\', 4)); torch.set_num_threads(NT); torch.manual_seed(0); np.random.seed(0)\nDATA = f\'{D}/v3/data\'\nTE, TR, EE, ER, NI, NU = load(DATA)\nSPEC = {**{f\'f{k}\': (J20 + k * DAY, DAY) for k in range(1, 7)}, \'val23\': (J20 + 3 * DAY, None), \'val25\': (J20 + 5 * DAY, None), \'val26\': (J20 + 6 * DAY, None), \'eval\': (J20 + 7 * DAY, \'eval\')}\nENC = os.environ.get(\'SQ_ENC\', \'din\'); HL, NP = int(os.environ.get(\'SQ_HL\', 20)), int(os.environ.get(\'SQ_NP\', 3)); SEED = int(os.environ.get(\'SQ_SEED\', 0)); TEMB = int(os.environ.get(\'SQ_TEMB\', 0)); HID = int(os.environ.get(\'SQ_HID\', 64)); SLATE = int(os.environ.get(\'SQ_SLATE\', 0))\ndef slates(EV, RW):\n """events EV (eid,u,ts,tb,dt,h,h_ts) + rows -> arrays in EV order; candidate matrix [E,10], clicks, rowids."""\n R = RW.join(EV.select(\'eid\', pl.int_range(pl.len()).alias(\'_e\')), on=\'eid\', how=\'inner\').sort(\'_e\', \'rowid\')\n e = R[\'_e\'].to_numpy(); assert (np.bincount(e, minlength=EV.height) == 10).all()\n cand = R[\'ii\'].to_numpy().reshape(-1, 10).astype(np.int64)\n clk = R[\'clicked\'].to_numpy().reshape(-1, 10).astype(np.float32) if \'clicked\' in R.columns else None\n return cand, clk, R[\'rowid\'].to_numpy().reshape(-1, 10)\n# global train index sorted by (u, ts, eid): used for prior impressions\nTS = TE.sort(\'u\', \'ts\', \'eid\'); T_u = TS[\'u\'].to_numpy().astype(np.int64); T_ts = TS[\'ts\'].to_numpy()\nT_cand, T_clk, _ = slates(TS, TR); KEY = T_u * (1 << 32) + T_ts\ndef tokens(EV, C):\n """[E, HL+NP*10] item / flag / gap-bucket tokens + ctx ints."""\n n = EV.height; u = EV[\'u\'].to_numpy().astype(np.int64); ts = EV[\'ts\'].to_numpy()\n it = np.full((n, HL + NP * 10), NI, np.int64); fl = np.zeros_like(it); gp = np.zeros_like(it)\n hh = EV[\'h\'].to_list(); ht = EV[\'h_ts\'].to_list()\n for i in range(n):\n l = hh[i]\n if l:\n l = l[-HL:]; t = np.asarray(ht[i][-HL:]); k = len(l)\n it[i, :k] = l; fl[i, :k] = 1; gp[i, :k] = 1 + np.clip(np.log2(np.maximum(ts[i] - t, 60) / 60), 0, 19).astype(np.int64)\n lim = np.minimum(ts, C); pos = np.searchsorted(KEY, u * (1 << 32) + lim, \'left\') - 1 # last train event with ts < lim\n npr = np.zeros(n, np.int64)\n for j in range(NP):\n p = pos - j; ok = (p >= 0) & (T_u[np.maximum(p, 0)] == u); p = np.maximum(p, 0)\n assert (T_ts[p][ok] < lim[ok]).all() and (T_ts[p][ok] < C).all()\n s = slice(HL + 10 * j, HL + 10 * (j + 1))\n it[:, s] = np.where(ok[:, None], T_cand[p], NI); fl[:, s] = np.where(ok[:, None], 2 + T_clk[p].astype(np.int64), 0)\n g = 1 + np.clip(np.log2(np.maximum(ts - T_ts[p], 60) / 60), 0, 19).astype(np.int64); gp[:, s] = np.where(ok, g, 0)[:, None]\n npr += ok\n ctx = np.stack([EV[\'tb\'].to_numpy().astype(np.int64), EV[\'dt\'].to_numpy().astype(np.int64), (fl[:, :HL] > 0).sum(1), npr], 1)\n return it, fl, gp, ctx\ndef ppmi_E(C, k=64):\n P_ev = TE.filter(pl.col(\'ts\') < C); P_rows = TR.join(P_ev.select(\'eid\', \'u\', \'ts\'), on=\'eid\', how=\'inner\')\n CLg = click_log(C, P_ev, P_rows, P_ev.head(0)); assert CLg[\'ts\'].max() < C\n s_ = CLg.select(\'u\', pl.col(\'ii\').cast(pl.Int64)).to_numpy(); u, it = s_[:, 0], s_[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32)\n for d in range(1, 6):\n m = u[d:] == u[:-d]; x, y = it[:-d][m], it[d:][m]; ok = x != y; x, y = x[ok], y[ok]; w = np.full(len(x), 1.0 / d, np.float32)\n A = A + sparse.csr_matrix((w, (x, y)), shape=(NI, NI)) + sparse.csr_matrix((w, (y, x)), shape=(NI, NI))\n A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum()\n pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0\n Mx = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI))\n U, S, _ = randomized_svd(Mx, k, random_state=0, n_iter=5); V = U * np.sqrt(S); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9)\n return np.vstack([V, np.zeros((1, k))]).astype(np.float32)\nclass M(nn.Module):\n def __init__(s, EMB, d=32, H=2, dh=16):\n super().__init__(); s.H, s.dh = H, dh\n s.E0 = nn.Embedding.from_pretrained(torch.from_numpy(EMB), freeze=True); s.P = nn.Linear(EMB.shape[1], d); s.S = nn.Embedding(2, H)\n s.T = nn.Embedding(EMB.shape[0], d) if TEMB else None\n if TEMB: nn.init.normal_(s.T.weight, std=0.01)\n s.E = (lambda x: s.P(s.E0(x)) + s.T(x)) if TEMB else (lambda x: s.P(s.E0(x))); s.F = nn.Embedding(4, d); s.G = nn.Embedding(21, d);\n s.c = nn.ModuleList([nn.Embedding(4, 4), nn.Embedding(2, 2), nn.Embedding(HL + 1, 4), nn.Embedding(NP + 1, 2)])\n s.q = nn.Linear(d, H * dh); s.k = nn.Linear(d, H * dh); s.v = nn.Linear(d, H * dh); s.null = nn.Parameter(torch.zeros(1, 1, H, dh))\n if ENC == \'rec\': s.R = nn.Embedding(21, d)\n if ENC == \'gru\': s.gru = nn.GRU(d, d, batch_first=True)\n s.mlp = nn.Sequential(nn.Linear(d * (1 + 2 * SLATE) + H * dh * (1 + SLATE) + 12, HID), nn.ReLU(), nn.Linear(HID, 1))\n def forward(s, it, fl, gp, ctx, cand, extra=False):\n B, T = it.shape; K = s.E(it) + s.F(fl) + s.G(gp); q = s.E(cand) # (B,T,d), (B,10,d)\n if ENC == \'rec\': K = K + s.R(gp) # recency-aware keys (extra gap emb)\n if ENC == \'gru\': # GRU over tokens in reverse-recency order -> contextualised keys\n K = K + s.gru(K)[0]\n Q = s.q(q).view(B, 10, s.H, s.dh); Kk = s.k(K).view(B, T, s.H, s.dh); V = s.v(K).view(B, T, s.H, s.dh)\n same = (cand[:, :, None] == it[:, None, :]).long(); att = torch.einsum(\'bchd,bthd->bcht\', Q, Kk) / s.dh ** 0.5 + s.S(same).permute(0, 1, 3, 2); att = att.masked_fill((fl == 0)[:, None, None, :], -1e9)\n an = (Q * s.null).sum(-1, keepdim=True) / s.dh ** 0.5; w = torch.softmax(torch.cat([att, an], -1), -1)\n P = torch.einsum(\'bcht,bthd->bchd\', w[..., :T], V).reshape(B, 10, -1)\n cx = torch.cat([e(ctx[:, i]) for i, e in enumerate(s.c)], -1)[:, None].expand(B, 10, 12)\n f = [q, P, cx]\n if SLATE: # within-event context: the slate\'s own candidates only\n qm = q.mean(1, keepdim=True).expand_as(q); Pm = P.mean(1, keepdim=True).expand_as(P); f = [q, q - qm, qm, P, P - Pm, cx]\n out = s.mlp(torch.cat(f, -1)).squeeze(-1)\n if not extra: return out\n valid = (fl > 0).float(); nv = valid.sum(1).clamp(min=1) # (B,T),(B,)\n wm = w.mean(2) # head-mean weights (B,10,T+1)\n a_max = wm[..., :T].max(-1).values; a_null = wm[..., T]; a_mean = (1 - a_null) / nv[:, None]\n ent = -(wm.clamp(min=1e-9).log() * wm).sum(-1)\n vq = s.v(q) # candidate in value space (B,10,H*dh)\n pdot = (P * vq).sum(-1); pcos = pdot / (P.norm(dim=-1) * vq.norm(dim=-1) + 1e-6)\n mu = (K * valid[..., None]).sum(1) / nv[:, None] # mean-pooled user emb (B,d)\n mdot = (q * mu[:, None]).sum(-1); mcos = mdot / (q.norm(dim=-1) * mu.norm(dim=-1)[:, None] + 1e-6)\n ck = ((fl == 1) | (fl == 3)).float(); e0 = s.E0(it); eu = (e0 * ck[..., None]).sum(1); ec = s.E0(cand)\n ecos = (ec * eu[:, None]).sum(-1) / (ec.norm(dim=-1) * eu.norm(dim=-1)[:, None] + 1e-6)\n return out, torch.stack([a_max, a_null, a_mean, ent, pdot, pcos, mdot, mcos, ecos], -1)\nXN = [\'sq_amax\', \'sq_anull\', \'sq_amean\', \'sq_aent\', \'sq_pdot\', \'sq_pcos\', \'sq_mdot\', \'sq_mcos\', \'sq_ecos\']\ndef usel(u): return (u * 2654435761 % 2 ** 32) / 2 ** 32 < a.frac\n_M = {}\ndef fit(C):\n if C in _M: return _M[C]\n torch.manual_seed(SEED); np.random.seed(SEED)\n t0 = time.time(); m = T_ts < C; idx = np.nonzero(m)[0]; idx = idx[np.argsort(T_ts[idx], kind=\'stable\')][-a.ntr:] # train on all users; --frac only subsamples target users\n it, fl, gp, ctx = tokens(TS[idx], C); cand = T_cand[idx]; y = T_clk[idx]; tk = time.time() - t0\n X = [torch.from_numpy(z) for z in (it, fl, gp, ctx, cand)]; Y = torch.from_numpy(y)\n te = time.time(); EMB = ppmi_E(C); print(f\' ppmi C={C} {time.time() - te:.0f}s\', flush=True)\n mdl = M(EMB, a.d, int(os.environ.get(\'SQ_H\', 2)), int(os.environ.get(\'SQ_DH\', 16))); opt = torch.optim.Adam(mdl.parameters(), float(os.environ.get(\'SQ_LR\', 2e-3))); lossf = nn.BCEWithLogitsLoss(); bs = int(os.environ.get(\'SQ_BS\', 256)); n = len(idx)\n maxst = int(os.environ.get(\'SQ_MAXSTEPS\', 6000)); st = 0 # v8: hard cap on optimizer steps -> bounded eval-time fit\n for ep in range(a.ep):\n perm = torch.randperm(n); tl = 0\n for b in range(0, n, bs):\n if st >= maxst: break\n st += 1; j = perm[b:b + bs]; opt.zero_grad(); l = lossf(mdl(*[x[j] for x in X]), Y[j]); l.backward(); opt.step(); tl += l.item() * len(j)\n print(f\' C={C} ep{ep} loss {tl / n:.4f} n={n} steps={st} {time.time() - t0:.0f}s\', flush=True)\n mdl.eval(); _M[C] = mdl; print(f\' fit C={C} tok {tk:.0f}s total {time.time() - t0:.0f}s\', flush=True); return mdl\ndef score(mdl, EV, C, cand):\n it, fl, gp, ctx = tokens(EV, C); out = []; ex = []\n with torch.no_grad():\n for b in range(0, len(cand), 8192):\n o, e = mdl(*[torch.from_numpy(z[b:b + 8192]) for z in (it, fl, gp, ctx, cand)], extra=True); out.append(o.numpy()); ex.append(e.numpy())\n return np.concatenate(out), np.concatenate(ex)\nTIM = {}\nfor name in a.names:\n t0 = time.time(); name, _, mn = name.partition(\'@\'); C, W = SPEC[name]; CM = SPEC[mn][0] if mn else C # \'f4@f3\': score f4 rows (history at C_f4) with the model fit at C_f3 < C_f4\n assert CM <= C\n if W == \'eval\': EV, RW = EE, ER\n else: EV = first_events(TE, C, W); RW = TR.join(EV.select(\'eid\'), on=\'eid\', how=\'semi\')\n if a.frac < 1: EV = EV.filter(pl.Series(usel(EV[\'u\'].to_numpy().astype(np.int64))))\n cand, _, rid = slates(EV, RW); S, X = score(fit(CM), EV, C, cand)\n z = (S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6); lse = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1)\n rk = (np.argsort(np.argsort(-S, 1, kind=\'stable\'), 1) + 1).astype(np.float32) # 1 = event top\n gap = S - S.max(1, keepdims=True); gap2 = S - np.sort(S, 1)[:, -2:-1] # gap to top / to runner-up\n cols = {\'rowid\': rid.ravel(), \'sq_logit\': S.ravel(), \'sq_z\': z.ravel(), \'sq_dm\': (S - S.mean(1, keepdims=True)).ravel(), \'sq_lse\': lse.ravel(),\n \'sq_rank\': rk.ravel(), \'sq_gap\': gap.ravel(), \'sq_gap2\': gap2.ravel()}\n for i, n_ in enumerate(XN): cols[n_] = X[..., i].ravel()\n for n_ in (\'sq_pcos\', \'sq_mcos\', \'sq_ecos\'): # within-event demeaned versions\n v = X[..., XN.index(n_)]; cols[n_ + \'_dm\'] = (v - v.mean(1, keepdims=True)).ravel()\n df = pl.DataFrame(cols).with_columns(pl.exclude(\'rowid\').cast(pl.Float32))\n df.write_parquet(f\'{a.out}/{name}.parquet\'); print(name, df.shape, f\'{time.time() - t0:.0f}s\', flush=True)\n', 'seqrun.py': '"""seqrun.py NAMES... : multi-seed sequence-model stage. Runs seq11.py once per seed (SQ_SEED = 0..V12_SQ_SEEDS-1) with at most V12_SQ_PAR runs\nat the same time, SQ_THREADS torch threads each, then averages the seeds into feat_sqC/ (avg_sq.py).\nSeeds 0-7: base config (== seq10); seeds 8-10: SQ_H=4 SQ_HID=128 (4 heads, hidden 128) for ensemble diversity.\nEnv: V12_SQ_SEEDS (default 11), V12_SQ_PAR (default min(3, threads//2)), V12_SQ_THREADS (torch threads per run, default threads//par)."""\nimport os, sys, time, subprocess\nW = \'@@WORK@@\'; names = sys.argv[1:]\nNS = int(os.environ.get(\'V12_SQ_SEEDS\', 11)); T = int(os.environ.get(\'V12_THREADS\', 8))\nPAR = int(os.environ.get(\'V12_SQ_PAR\', max(1, min(3, T // 2)))); TH = int(os.environ.get(\'V12_SQ_THREADS\', max(1, T // PAR)))\nprint(f\'seqrun: {NS} seeds, {PAR} concurrent, {TH} threads each, tables {names}\', flush=True)\nassert NS >= 1\ndef cfg(s): return {\'SQ_H\': \'4\', \'SQ_HID\': \'128\'} if s >= 8 else {}\ntodo = list(range(NS)); run = {}; t0 = time.time(); done = []\nos.makedirs(f\'{W}/msC/logs\', exist_ok=True)\ndef finished(s): return all(os.path.exists(f\'{W}/msC/s{s}/{n}.parquet\') for n in names) and os.path.exists(f\'{W}/msC/s{s}/DONE\')\ntodo = [s for s in todo if not finished(s)]; print(\'seeds to run\', todo, flush=True)\nwhile todo or run:\n while todo and len(run) < PAR:\n s = todo.pop(0); od = f\'{W}/msC/s{s}\'; os.makedirs(od, exist_ok=True)\n env = dict(os.environ, SQ_SEED=str(s), SQ_THREADS=str(TH), SQ_OUT=od, SQ_WORK=W, SQ_NTR=\'500000\', POLARS_MAX_THREADS=str(TH), OMP_NUM_THREADS=str(TH), OPENBLAS_NUM_THREADS=str(TH), MKL_NUM_THREADS=str(TH), PYTHONUNBUFFERED=\'1\', **cfg(s))\n lf = open(f\'{W}/msC/logs/s{s}.log\', \'w\')\n run[s] = (subprocess.Popen([sys.executable, \'seq11.py\'] + names, cwd=W, env=env, stdout=lf, stderr=subprocess.STDOUT), time.time()); print(f\'[{time.time() - t0:.0f}s] start seed {s}\', flush=True)\n time.sleep(15) # stagger the memory-heavy start-up\n time.sleep(3)\n for s, (p, ts) in list(run.items()):\n r = p.poll()\n if r is None: continue\n del run[s]\n if r != 0:\n for q, _ in run.values(): q.kill()\n sys.exit(f\'seq seed {s} failed (rc {r}); see {W}/msC/logs/s{s}.log\')\n open(f\'{W}/msC/s{s}/DONE\', \'w\').write(\'ok\'); print(f\'[{time.time() - t0:.0f}s] seed {s} done in {time.time() - ts:.0f}s\', flush=True)\nsubprocess.run([sys.executable, \'avg_sq.py\', f\'{W}/feat_sqC\'] + [f\'{W}/msC/s{s}\' for s in range(NS)], cwd=W, check=True)\nprint(f\'seqrun done {time.time() - t0:.0f}s\', flush=True)\n', 'ttfeat.py': '"""data_dig_v2 features, refit per cutoff C on train only (events ts < C); eval fitted at J27 on all train.\n(A) row-scaled exact SVD: binary user x item of ALL pre-C history clicks, diag(1/sqrt(len_u)) M diag(1/sqrt(pop+5)), svds k16 rs0,\n comps 2..13 * s -> tA_cos = cos(candidate, own-history centroid) (+ _z, _dmax within event).\n(B) two-tower conditional logit, tied item vectors initialised from L2-normalised (A) rows, u = mean of last<=20 own-history vectors,\n score u.v, 10-way softmax CE on clicked train events (ts 0\n cand = torch.from_numpy(np.array(G[\'ii\'].to_list(), np.int64)[keep]); lab = torch.from_numpy(clk[keep].argmax(1).astype(np.int64))\n Ht = torch.from_numpy(hmat(G.filter(pl.Series(keep))))\n torch.manual_seed(0); np.random.seed(0)\n P = torch.nn.Embedding(NI + 1, 12, padding_idx=NI)\n with torch.no_grad(): P.weight[:NI] = torch.from_numpy(nrm(VA)); P.weight[NI] = 0\n opt = torch.optim.Adam(P.parameters(), lr=1e-3); N = len(lab)\n def uvec(H):\n m = (H < NI).float(); return (P(H) * m[..., None]).sum(1) / m.sum(1, keepdim=True).clamp(min=1)\n for ep in range(6):\n perm = torch.randperm(N); tot = 0.0\n for i in range(0, N, 2048):\n j = perm[i:i + 2048]; sc = (uvec(Ht[j])[:, None, :] * P(cand[j])).sum(-1)\n loss = torch.nn.functional.cross_entropy(sc, lab[j]); opt.zero_grad(); loss.backward(); opt.step()\n with torch.no_grad(): P.weight[NI] = 0\n tot += loss.item() * len(j)\n print(f\' C {C} ep{ep} loss {tot / N:.4f}\', flush=True)\n VB = P.weight.detach().numpy().copy()\n print(time.strftime(\'%T\'), \'C\', C, \'train events\', N, f\'{time.time() - t0:.0f}s\', flush=True)\n _M[C] = (VA, VB); return _M[C]\ndef build(C, T_ev, T_rows):\n VA, VB = fit(C)\n E = T_ev.select(\'eid\', \'h\').with_row_index(\'k\')\n R = T_rows.select(\'rowid\', \'eid\', \'ii\').join(E.select(\'eid\', \'k\'), on=\'eid\', how=\'left\')\n rk = R[\'k\'].to_numpy(); ri = R[\'ii\'].to_numpy().astype(np.int64)\n # (A) all-history centroid cosine\n V = nrm(VA); H = E.select(\'k\', \'h\').explode(\'h\').drop_nulls(\'h\'); hc = np.asarray(sp.csr_matrix((np.ones(H.height), (H[\'k\'].to_numpy(), H[\'h\'].to_numpy().astype(np.int64))), shape=(E.height, V.shape[0])) @ V) # v8: replaces np.add.at\n hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1)\n a = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan)\n # (B) tower dot with u = mean of last <= 20 own-history vectors\n Hm = hmat(E); m = (Hm < NI); Vp = np.vstack([VB[:NI], np.zeros((1, VB.shape[1]), VB.dtype)])\n u = (Vp[Hm] * m[..., None]).sum(1) / np.maximum(m.sum(1, keepdims=True), 1)\n dot = np.where(m.sum(1)[rk] > 0, (u[rk] * VB[ri]).sum(1), np.nan)\n R = R.select(\'rowid\', \'eid\').with_columns(pl.Series(\'tA_cos\', a.astype(np.float32)).fill_nan(None), pl.Series(\'tB_dot\', dot.astype(np.float32)).fill_nan(None))\n R = R.with_columns(((pl.col(\'tA_cos\') - pl.col(\'tA_cos\').mean().over(\'eid\')) / (pl.col(\'tA_cos\').std().over(\'eid\') + 1e-6)).alias(\'tA_z\'),\n (pl.col(\'tA_cos\') - pl.col(\'tA_cos\').max().over(\'eid\')).alias(\'tA_dmax\'),\n pl.col(\'tB_dot\').exp().sum().over(\'eid\').log().alias(\'tB_lse\'), pl.col(\'tB_dot\').mean().over(\'eid\').alias(\'tB_mean\'),\n (pl.col(\'tB_dot\') - pl.col(\'tB_dot\').mean().over(\'eid\')).alias(\'tB_dmean\'),\n ((pl.col(\'tB_dot\') - pl.col(\'tB_dot\').mean().over(\'eid\')) / (pl.col(\'tB_dot\').std().over(\'eid\') + 1e-6)).alias(\'tB_dot_z\'))\n return R.drop(\'eid\')\nfor n in sys.argv[1:]:\n t0 = time.time()\n if n == \'eval\': R = build(J20 + 7 * DAY, EE, ER)\n else:\n C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select(\'eid\'), on=\'eid\', how=\'semi\'))\n R.write_parquet(f\'{OUT}/{n}.parquet\'); print(n, R.shape, f\'{time.time() - t0:.0f}s\', flush=True)\n', 'v3/features.py': '"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR.\n\nbuild(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate.\nEvery statistic uses ONLY:\n * train events/rows with ts < C (fit window: impressions + labels), and\n * click histories of events with ts < C, plus the target event\'s OWN history (strictly before its ts).\nSince target events are each user\'s first event at/after C, their histories contain only clicks < C,\nso nothing observed after a target row\'s own event time is ever joined.\n\nFeature groups (column prefixes, used for ablation):\n hist_ candidate vs the user\'s click log / own history / past impressions of that user-item pair\n it_ item popularity & CTR (full fit window, last 24h, last 6h, trend)\n co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user\'s last 5/20 clicks\n us_ user activity / CTR / recency / cold-start flags\n ev_ within-event rank / z-score / event max & mean of the main features\n tm_ time_bucket, day_type, hour, staleness since cutoff\n sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines\n mf_ logistic matrix factorisation score fitted on fit-window impressions\n"""\nimport time\nimport numpy as np\nimport polars as pl\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\n\nJ20 = 1737331200\nDAY = 86400\nG_PRIOR = 0.0629\nK_ITEM = 20 # pseudo-impressions for item CTR shrinkage\nK_USER = 5 # pseudo-events for user CTR shrinkage\nTOPK = 200 # co-click neighbours kept per item\nID_COLS = [\'rowid\', \'eid\', \'u\', \'ii\', \'ts\', \'clicked\', \'ord\']\n_t0 = time.time()\n\ndef log(*a):\n print(time.strftime(\'%H:%M:%S\'), *a, flush=True)\n\ndef load(D):\n TE = pl.read_parquet(f\'{D}/ev_train.parquet\')\n TR = pl.read_parquet(f\'{D}/rows_train.parquet\')\n EE = pl.read_parquet(f\'{D}/ev_eval.parquet\')\n ER = pl.read_parquet(f\'{D}/rows_eval.parquet\')\n NI = pl.read_parquet(f\'{D}/items.parquet\').height\n NU = pl.read_parquet(f\'{D}/users.parquet\').height\n return TE, TR, EE, ER, NI, NU\n\ndef _hist_long(E):\n return (E.select(\'u\', \'h\', \'h_ts\').explode([\'h\', \'h_ts\']).drop_nulls(\'h\')\n .select(\'u\', pl.col(\'h\').cast(pl.Int32).alias(\'ii\'), pl.col(\'h_ts\').cast(pl.Int64).alias(\'ts\')))\n\ndef click_log(C, P_ev, P_rows, T_ev):\n """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories."""\n win = P_rows.filter(pl.col(\'clicked\') == 1).select(\'u\', pl.col(\'ii\').cast(pl.Int32), \'ts\')\n hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col(\'ts\') < C)\n CL = pl.concat([win, hist]).unique().sort(\'u\', \'ts\', \'ii\')\n return CL\n\ndef cooc_tables(CL, NI):\n """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next."""\n us = CL.select(\'u\', \'ii\').unique()\n pop = us.group_by(\'ii\').agg(pl.len().alias(\'pop\'))\n pairs = (us.join(us, on=\'u\', suffix=\'_b\').filter(pl.col(\'ii\') != pl.col(\'ii_b\'))\n .group_by(\'ii\', \'ii_b\').agg(pl.len().cast(pl.Int32).alias(\'co\')))\n co = (pairs.join(pop, on=\'ii\').join(pop.rename({\'ii\': \'ii_b\', \'pop\': \'pop_b\'}), on=\'ii_b\')\n .with_columns((pl.col(\'co\') / (pl.col(\'pop\') * pl.col(\'pop_b\')).sqrt()).cast(pl.Float32).alias(\'cos\'))\n .select(pl.col(\'ii_b\').alias(\'j\'), \'ii\', \'cos\') # j = history item, ii = candidate\n .sort(\'cos\', descending=True).group_by(\'j\').head(TOPK))\n seq = CL.select(\'u\', \'ii\') # already sorted by u, ts\n parts = []\n for lag in (1, 2, 3):\n parts.append(seq.with_columns(pl.col(\'ii\').shift(lag).over(\'u\').alias(\'j\')).drop_nulls(\'j\')\n .select(pl.col(\'j\').cast(pl.Int32), \'ii\', pl.lit(1.0 / lag).alias(\'w\')))\n trn = pl.concat(parts).filter(pl.col(\'j\') != pl.col(\'ii\')).group_by(\'j\', \'ii\').agg(pl.col(\'w\').sum())\n tot = trn.group_by(\'j\').agg(pl.col(\'w\').sum().alias(\'tot\'))\n trn = (trn.join(tot, on=\'j\').with_columns((pl.col(\'w\') / (pl.col(\'tot\') + 5.0)).cast(pl.Float32).alias(\'trp\'))\n .select(\'j\', \'ii\', \'trp\').sort(\'trp\', descending=True).group_by(\'j\').head(TOPK))\n return co, trn\n\ndef fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000):\n """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy)."""\n rng = np.random.default_rng(seed)\n u = P_rows[\'u\'].to_numpy().astype(np.int64); i = P_rows[\'ii\'].to_numpy().astype(np.int64)\n y = P_rows[\'clicked\'].to_numpy().astype(np.float32)\n P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32)\n bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32)\n b = np.float32(np.log(y.mean() / (1 - y.mean())))\n gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6)\n n = len(y)\n for ep in range(epochs):\n perm = rng.permutation(n)\n for s in range(0, n, batch):\n idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx]\n pu, qi = P[ub], Q[ib]\n z = b + bu[ub] + bi[ib] + (pu * qi).sum(1)\n g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz\n m = len(idx)\n Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m))\n Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m))\n dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None]\n dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None]\n dbu = Su @ g; dbi = Si @ g\n gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi\n P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ)\n bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi)\n return dict(b=b, bu=bu, bi=bi, P=P, Q=Q)\n\ndef build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode=\'std\'):\n t0 = time.time()\n P_ev = TE.filter(pl.col(\'ts\') < C)\n P_rows = TR.join(P_ev.select(\'eid\', \'u\', \'ts\'), on=\'eid\', how=\'inner\')\n g = float(P_rows[\'clicked\'].mean())\n ndays = (C - J20) / DAY\n CL = click_log(C, P_ev, P_rows, T_ev)\n\n # ---------------- item ----------------\n def ictr(c, n):\n return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM)\n it = P_rows.group_by(\'ii\').agg(\n pl.len().alias(\'it_imp\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'it_clk\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'it_imp24\'), pl.col(\'clicked\').filter(pl.col(\'ts\') >= C - DAY).sum().cast(pl.Int32).alias(\'it_clk24\'),\n (pl.col(\'ts\') >= C - 6 * 3600).sum().alias(\'it_imp6\'), pl.col(\'clicked\').filter(pl.col(\'ts\') >= C - 6 * 3600).sum().cast(pl.Int32).alias(\'it_clk6\'))\n it = it.with_columns(ictr(\'it_clk\', \'it_imp\').alias(\'it_ctr\'), ictr(\'it_clk24\', \'it_imp24\').alias(\'it_ctr24\'),\n ictr(\'it_clk6\', \'it_imp6\').alias(\'it_ctr6\'))\n it = it.with_columns((pl.col(\'it_ctr24\') / pl.col(\'it_ctr\')).alias(\'it_trend24\'), (pl.col(\'it_ctr6\') / pl.col(\'it_ctr\')).alias(\'it_trend6\'),\n (pl.col(\'it_imp24\') / (pl.col(\'it_imp\') / ndays)).alias(\'it_imptrend24\'),\n (pl.col(\'it_imp\') / ndays).alias(\'it_imp_rate\'))\n ip = CL.group_by(\'ii\').agg(pl.len().alias(\'it_nclk_all\'), pl.col(\'u\').n_unique().alias(\'it_nusers\'),\n (pl.col(\'ts\') < J20).sum().alias(\'it_nclk_warm\'),\n ((pl.col(\'ts\') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias(\'it_clk_dec2d\'))\n\n # ---------------- user ----------------\n ue = P_rows.group_by(\'u\').agg(pl.col(\'eid\').n_unique().alias(\'us_nev\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'us_nclk_win\'),\n pl.col(\'ts\').max().alias(\'_u_last_ev\'))\n ue = ue.with_columns(((pl.col(\'us_nclk_win\') + K_USER * 0.629) / (pl.col(\'us_nev\') + K_USER)).alias(\'us_ctr\'))\n uc = CL.group_by(\'u\').agg(pl.len().alias(\'us_nclk_all\'), (pl.col(\'ts\') < J20).sum().alias(\'us_nclk_warm\'),\n pl.col(\'ii\').n_unique().alias(\'us_nuniq\'), pl.col(\'ts\').max().alias(\'_u_last_clk\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'us_nclk24\'))\n uc = uc.with_columns((1 - pl.col(\'us_nuniq\') / pl.col(\'us_nclk_all\')).alias(\'us_repeat_rate\'))\n\n # ---------------- user x item ----------------\n uic = CL.group_by(\'u\', \'ii\').agg(pl.len().alias(\'hist_n_all\'), (pl.col(\'ts\') < J20).sum().alias(\'hist_n_warm\'),\n (pl.col(\'ts\') >= J20).sum().alias(\'hist_n_win\'), pl.col(\'ts\').max().alias(\'_ui_last_clk\'))\n uii = P_rows.group_by(\'u\', \'ii\').agg(pl.len().alias(\'hist_imp\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'_ui_clk\'),\n pl.col(\'ts\').max().alias(\'_ui_last_imp\'))\n uii = uii.with_columns((pl.col(\'hist_imp\') - pl.col(\'_ui_clk\')).alias(\'hist_noclk\'))\n\n # ---------------- assemble ----------------\n R = (T_rows.select([c for c in (\'rowid\', \'eid\', \'ii\', \'clicked\', \'ord\') if c in T_rows.columns])\n .join(T_ev.select(\'eid\', \'u\', \'ts\', \'tb\', \'dt\', \'h\', \'h_ts\'), on=\'eid\', how=\'left\').with_row_index(\'r\'))\n R = (R.join(it, on=\'ii\', how=\'left\').join(ip, on=\'ii\', how=\'left\').join(ue, on=\'u\', how=\'left\').join(uc, on=\'u\', how=\'left\')\n .join(uic, on=[\'u\', \'ii\'], how=\'left\').join(uii, on=[\'u\', \'ii\'], how=\'left\'))\n R = R.with_columns(\n pl.col(\'h\').list.len().alias(\'us_hlen\'),\n (pl.col(\'h\').list.len() == 0).cast(pl.Int8).alias(\'us_nohist\'),\n pl.col(\'us_nev\').is_null().cast(pl.Int8).alias(\'us_cold\'),\n (pl.col(\'ts\') - pl.col(\'_u_last_ev\')).alias(\'us_since_ev\'),\n (pl.col(\'ts\') - pl.col(\'_u_last_clk\')).alias(\'us_since_clk\'),\n (pl.col(\'ts\') - pl.col(\'_ui_last_clk\')).alias(\'hist_since_clk\'),\n (pl.col(\'ts\') - pl.col(\'_ui_last_imp\')).alias(\'hist_since_imp\'),\n (pl.col(\'ts\') - C).alias(\'tm_stale\'), ((pl.col(\'ts\') // 3600) % 24).cast(pl.Int8).alias(\'tm_hour\'),\n pl.col(\'tb\').alias(\'tm_tb\'), pl.col(\'dt\').alias(\'tm_dt\'))\n # position (from end, 1 = most recent) of candidate in own history\n hx = (R.select(\'r\', \'ii\', \'h\').with_columns(pl.int_ranges(pl.col(\'h\').list.len(), 0, -1).alias(\'pos\'))\n .explode([\'h\', \'pos\']).drop_nulls(\'h\').filter(pl.col(\'h\') == pl.col(\'ii\'))\n .group_by(\'r\').agg(pl.col(\'pos\').min().alias(\'hist_pos\'), pl.len().alias(\'hist_n_own\')))\n R = R.join(hx, on=\'r\', how=\'left\')\n R = R.with_columns(pl.col(\'hist_since_clk\').fill_null(30 * DAY), pl.col(\'hist_since_imp\').fill_null(30 * DAY),\n pl.col(\'hist_pos\').fill_null(99), pl.col(\'hist_n_own\').fill_null(0), pl.col(\'us_since_ev\').fill_null(30 * DAY), pl.col(\'us_since_clk\').fill_null(30 * DAY))\n R = R.with_columns((pl.col(\'hist_n_all\').fill_null(0) > 0).cast(pl.Int8).alias(\'hist_in\'))\n\n # ---------------- co-click ----------------\n co, trn = cooc_tables(CL, NI)\n last20 = CL.group_by(\'u\', maintain_order=True).agg(pl.col(\'ii\').tail(20).alias(\'l20\'))\n T_u = R.select(\'r\', \'u\', \'ii\').join(last20, on=\'u\', how=\'left\')\n X = (T_u.with_columns(pl.int_ranges(pl.col(\'l20\').list.len(), 0, -1).alias(\'pos\')).explode([\'l20\', \'pos\'])\n .drop_nulls(\'l20\').rename({\'l20\': \'j\'}).with_columns(pl.col(\'j\').cast(pl.Int32))\n .join(co, on=[\'j\', \'ii\'], how=\'left\').join(trn, on=[\'j\', \'ii\'], how=\'left\')\n .with_columns(pl.col(\'cos\').fill_null(0), pl.col(\'trp\').fill_null(0)))\n l5 = pl.col(\'pos\') <= 5\n agg = X.group_by(\'r\').agg(\n pl.col(\'cos\').max().alias(\'co_cos_max20\'), pl.col(\'cos\').mean().alias(\'co_cos_mean20\'),\n pl.col(\'cos\').filter(l5).max().alias(\'co_cos_max5\'), pl.col(\'cos\').filter(l5).mean().alias(\'co_cos_mean5\'),\n pl.col(\'cos\').filter(pl.col(\'pos\') == 1).first().alias(\'co_cos_last\'),\n pl.col(\'trp\').max().alias(\'co_tr_max20\'), pl.col(\'trp\').mean().alias(\'co_tr_mean20\'),\n pl.col(\'trp\').filter(l5).max().alias(\'co_tr_max5\'), pl.col(\'trp\').filter(l5).mean().alias(\'co_tr_mean5\'),\n pl.col(\'trp\').filter(pl.col(\'pos\') == 1).first().alias(\'co_tr_last\'),\n (pl.col(\'cos\') / pl.col(\'pos\').cast(pl.Float32).sqrt()).sum().alias(\'co_cos_rw\'), pl.col(\'cos\').sum().alias(\'co_cos_sum20\'))\n R = R.join(agg, on=\'r\', how=\'left\').sort(\'r\')\n\n # ---------------- SVD embeddings ----------------\n # sv_mode (leakage checks only): \'std\' = click log as of C; \'notgt\' = drop target events\' own histories;\n # \'lag1d\' = only clicks/impressions with ts < C - 1 day\n if sv_mode == \'std\':\n CLsv, NEGsrc = CL, P_rows\n elif sv_mode == \'notgt\':\n CLsv = pl.concat([P_rows.filter(pl.col(\'clicked\') == 1).select(\'u\', pl.col(\'ii\').cast(pl.Int32), \'ts\'),\n _hist_long(P_ev).filter(pl.col(\'ts\') < C)]).unique()\n NEGsrc = P_rows\n elif sv_mode == \'lag1d\':\n CLsv, NEGsrc = CL.filter(pl.col(\'ts\') < C - DAY), P_rows.filter(pl.col(\'ts\') < C - DAY)\n else:\n raise ValueError(sv_mode)\n assert (CLsv[\'ts\'].max() or 0) < C and (NEGsrc[\'ts\'].max() or 0) < C\n cnt = CLsv.group_by(\'u\', \'ii\').agg(pl.len().alias(\'n\'))\n Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy())), shape=(NU, NI))\n df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)\n Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc\n uu = R[\'u\'].to_numpy().astype(np.int64); iic = R[\'ii\'].to_numpy().astype(np.int64)\n ex = X.select(\'r\', \'j\', \'pos\').unique([\'r\', \'j\', \'pos\'])\n # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway\n er_r = ex[\'r\'].to_numpy(); er_j = ex[\'j\'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex[\'pos\'].to_numpy())).astype(np.float32)\n er_l5 = ex[\'pos\'].to_numpy() <= 5\n nR = R.height\n svcols = {}\n for k in (16, 32, 64):\n U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)\n V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = (U * np.sqrt(S)).astype(np.float32)\n Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9)\n cand = Vn[iic]\n svcols[f\'sv_k{k}_ucos\'] = (Un[uu] * cand).sum(1)\n svcols[f\'sv_k{k}_udot\'] = (Uf[uu] * V[iic]).sum(1)\n hv = np.zeros((nR, k), np.float32); np.add.at(hv, er_r, Vn[er_j] * er_w[:, None])\n hn = np.linalg.norm(hv, axis=1); svcols[f\'sv_k{k}_hcos\'] = (hv * cand).sum(1) / (hn + 1e-9)\n if k == 32:\n svcols[\'sv_k32_hnorm\'] = hn\n sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1)\n mx = np.full(nR, -1.0, np.float32); np.maximum.at(mx, er_r, sims)\n mx5 = np.full(nR, -1.0, np.float32); np.maximum.at(mx5, er_r[er_l5], sims[er_l5])\n svcols[\'sv_k32_imax20\'] = mx; svcols[\'sv_k32_imax5\'] = mx5\n if k == 64:\n # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only)\n Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy())), shape=(NU, NI))\n neg = NEGsrc.filter(pl.col(\'clicked\') == 0)\n Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg[\'u\'].to_numpy(), neg[\'ii\'].to_numpy())), shape=(NU, NI))\n nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9\n nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9\n cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1)\n svcols[\'sv_k64_ccos\'] = cc; svcols[\'sv_k64_ncos\'] = cn\n for a, nm in ((0.25, \'025\'), (0.5, \'05\'), (1.0, \'1\')):\n M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9\n svcols[f\'sv_k64_con{nm}\'] = (M[uu] * cand).sum(1)\n R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()])\n\n # ---------------- MF ----------------\n if with_mf:\n mf = fit_mf(P_rows, NU, NI)\n uu = R[\'u\'].to_numpy().astype(np.int64); ii = R[\'ii\'].to_numpy().astype(np.int64)\n dot = (mf[\'P\'][uu] * mf[\'Q\'][ii]).sum(1)\n R = R.with_columns(pl.Series(\'mf_score\', mf[\'b\'] + mf[\'bu\'][uu] + mf[\'bi\'][ii] + dot),\n pl.Series(\'mf_dot\', dot), pl.Series(\'mf_bi\', mf[\'bi\'][ii]))\n\n fill0 = [c for c in R.columns if c.startswith((\'it_imp\', \'it_clk\', \'it_n\', \'us_n\', \'hist_n\', \'hist_imp\', \'hist_noclk\', \'co_\')) and R[c].dtype != pl.List]\n R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])\n R = R.with_columns([pl.col(c).fill_null(g) for c in (\'it_ctr\', \'it_ctr24\', \'it_ctr6\')] +\n [pl.col(c).fill_null(1.0) for c in (\'it_trend24\', \'it_trend6\', \'it_imptrend24\')] +\n [pl.col(\'us_ctr\').fill_null(g), pl.col(\'it_imp_rate\').fill_null(0), pl.col(\'us_repeat_rate\').fill_null(0)])\n\n # ---------------- event context ----------------\n main = [\'it_ctr\', \'it_ctr24\', \'it_imp\', \'it_nclk_all\', \'it_clk_dec2d\', \'hist_n_all\', \'hist_noclk\',\n \'co_cos_max20\', \'co_cos_rw\', \'co_tr_max5\', \'co_tr_last\',\n \'sv_k16_ucos\', \'sv_k32_ucos\', \'sv_k64_ucos\', \'sv_k32_hcos\', \'sv_k64_hcos\', \'sv_k32_imax20\', \'sv_k64_con05\', \'sv_k64_con025\', \'sv_k64_ccos\'] + ([\'mf_score\', \'mf_dot\'] if with_mf else [])\n exprs = []\n for c in main:\n x = pl.col(c).cast(pl.Float64)\n exprs += [x.rank(\'average\').over(\'eid\').alias(f\'ev_{c}_rk\'),\n ((x - x.mean().over(\'eid\')) / (x.std().over(\'eid\') + 1e-9)).alias(f\'ev_{c}_z\'),\n x.max().over(\'eid\').alias(f\'ev_{c}_max\'), x.mean().over(\'eid\').alias(f\'ev_{c}_mean\')]\n R = R.with_columns(exprs)\n R = R.drop([c for c in R.columns if c.startswith(\'_\')] + [\'h\', \'h_ts\', \'tb\', \'dt\']).sort(\'r\').drop(\'r\')\n log(f\'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s\')\n return R\n\nGROUPS = [\'it\', \'hist\', \'us\', \'co\', \'ev\', \'tm\', \'sv\', \'mf\']\n\ndef feature_cols(df, groups):\n cols = []\n for c in df.columns:\n if c in ID_COLS: continue\n gname = c.split(\'_\')[0]\n if gname == \'ev\':\n src = c[3:].split(\'_\')[0] # ev__...\n if \'ev\' in groups and src in groups: cols.append(c)\n elif gname in groups:\n cols.append(c)\n return cols\n', 'v3/prep.py': '"""Parse train/evaluation CSVs into compact parquet tables (dense int ids).\n\nOutputs in v3/data/:\n rows_train.parquet : rowid, eid, ii (item idx), clicked\n rows_eval.parquet : rowid, eid, ii, ord (original row order)\n ev_train.parquet / ev_eval.parquet : eid, u (user idx), ts, tb, dt, h (list[int item idx]), h_ts (list[int])\n items.parquet / users.parquet : id string -> idx\nUsage: python prep.py [DATA_DIR]\n"""\nimport polars as pl, sys, time, os\nDATA = sys.argv[1] if len(sys.argv) > 1 else \'@@RAW@@\'\nOUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), \'data\')\nos.makedirs(OUT, exist_ok=True)\nt0 = time.time()\ndef log(*a): print(f\'[{time.time()-t0:6.1f}s]\', *a, flush=True)\n\ndef scan(name):\n return pl.scan_csv(f\'{DATA}/{name}.csv\', schema_overrides={\'history_item_ids\': pl.Utf8, \'history_timestamps\': pl.Utf8,\n \'rowid\': pl.Utf8, \'event_id\': pl.Utf8, \'user_id\': pl.Utf8, \'item_id\': pl.Utf8})\n\nraw = {}\nfor name, lab in ((\'train\', True), (\'evaluation\', False)):\n lf = scan(name)\n cols = [\'rowid\', \'event_id\', \'item_id\'] + ([\'clicked\'] if lab else [])\n rows = lf.select(cols).collect(engine=\'streaming\')\n ev = (lf.select(\'event_id\', \'user_id\', \'timestamp\', \'time_bucket\', \'day_type\', \'history_item_ids\', \'history_timestamps\')\n .unique(subset=\'event_id\', keep=\'any\')\n .with_columns(pl.col(\'history_item_ids\').str.json_decode(pl.List(pl.Utf8)).alias(\'h_str\'),\n pl.col(\'history_timestamps\').str.json_decode(pl.List(pl.Int64)).alias(\'h_ts\'))\n .drop(\'history_item_ids\', \'history_timestamps\').collect(engine=\'streaming\'))\n raw[name] = (rows, ev)\n log(name, rows.shape, ev.shape)\n\ntr_rows, tr_ev = raw[\'train\']; ev_rows, ev_ev = raw[\'evaluation\']\nitems = pl.concat([tr_rows[\'item_id\'], ev_rows[\'item_id\'], tr_ev[\'h_str\'].explode().drop_nulls(),\n ev_ev[\'h_str\'].explode().drop_nulls()]).unique().sort()\nitems = pl.DataFrame({\'item_id\': items, \'ii\': pl.arange(0, len(items), eager=True).cast(pl.Int32)})\nusers = pl.concat([tr_ev[\'user_id\'], ev_ev[\'user_id\']]).unique().sort()\nusers = pl.DataFrame({\'user_id\': users, \'u\': pl.arange(0, len(users), eager=True).cast(pl.Int32)})\nevents = pl.concat([tr_ev[\'event_id\'], ev_ev[\'event_id\']]).unique().sort()\nevents = pl.DataFrame({\'event_id\': events, \'eid\': pl.arange(0, len(events), eager=True).cast(pl.Int32)})\nlog(\'items\', len(items), \'users\', len(users), \'events\', len(events))\nitems.write_parquet(f\'{OUT}/items.parquet\'); users.write_parquet(f\'{OUT}/users.parquet\')\n\ndef map_rows(rows, lab):\n r = rows.with_row_index(\'ord\').join(events, on=\'event_id\', how=\'left\').join(items, on=\'item_id\', how=\'left\')\n sel = [\'ord\', \'rowid\', \'eid\', \'ii\'] + ([\'clicked\'] if lab else [])\n r = r.sort(\'ord\').select(sel)\n if lab: r = r.with_columns(pl.col(\'clicked\').cast(pl.Int8))\n return r\n\ndef map_ev(ev):\n e = ev.join(events, on=\'event_id\', how=\'left\').join(users, on=\'user_id\', how=\'left\')\n ex = e.select(\'eid\', \'h_str\').with_columns(pl.int_ranges(0, pl.col(\'h_str\').list.len()).alias(\'k\')).explode([\'h_str\', \'k\']).drop_nulls(\'h_str\')\n ex = ex.join(items.rename({\'item_id\': \'h_str\'}), on=\'h_str\', how=\'left\').sort(\'eid\', \'k\')\n h = ex.group_by(\'eid\', maintain_order=True).agg(pl.col(\'ii\').alias(\'h\'))\n e = e.join(h, on=\'eid\', how=\'left\').with_columns(pl.col(\'h\').fill_null(pl.lit([], dtype=pl.List(pl.Int32))))\n return e.select(\'eid\', \'u\', pl.col(\'timestamp\').cast(pl.Int64).alias(\'ts\'), pl.col(\'time_bucket\').cast(pl.Int8).alias(\'tb\'),\n pl.col(\'day_type\').cast(pl.Int8).alias(\'dt\'), \'h\', \'h_ts\').sort(\'ts\', \'eid\')\n\nmap_rows(tr_rows, True).drop(\'ord\').write_parquet(f\'{OUT}/rows_train.parquet\')\nmap_rows(ev_rows, False).write_parquet(f\'{OUT}/rows_eval.parquet\')\nmap_ev(tr_ev).write_parquet(f\'{OUT}/ev_train.parquet\')\nmap_ev(ev_ev).write_parquet(f\'{OUT}/ev_eval.parquet\')\nlog(\'done\')\n', 'v3/split.py': '"""Time-based splits that mirror the official evaluation.\n\nEvaluation = each user\'s FIRST event at/after the cutoff. So for a cutoff C:\n fit events = all train events with ts < C (labels usable for aggregates)\n target/val = for each user, the first train event with C <= ts < C+W (W=None -> until end of train)\nWhole events always stay together (split is by event, rows follow their event).\n"""\nimport polars as pl\n\nJ20 = 1737331200 # 2025-01-20 00:00 UTC (start of train window; earlier clicks = warm-up)\nDAY = 86400\nC_MAIN = J20 + 5 * DAY # 2025-01-25\nC_SECOND = J20 + 6 * DAY # 2025-01-26\nC_FINAL = J20 + 7 * DAY # 2025-01-27 (official cutoff)\n\ndef fit_events(TE: pl.DataFrame, C: int) -> pl.DataFrame:\n return TE.filter(pl.col(\'ts\') < C)\n\ndef first_events(TE: pl.DataFrame, C: int, W: int | None = None) -> pl.DataFrame:\n """First event per user in [C, C+W). Ties on ts broken by eid (deterministic)."""\n m = pl.col(\'ts\') >= C\n if W is not None:\n m = m & (pl.col(\'ts\') < C + W)\n return TE.filter(m).sort(\'ts\', \'eid\').group_by(\'u\', maintain_order=True).first()\n\ndef split(TE: pl.DataFrame, TR: pl.DataFrame, C: int, W: int | None = None):\n """Return (fit_ev, fit_rows, val_ev, val_rows)."""\n fe = fit_events(TE, C)\n ve = first_events(TE, C, W)\n fr = TR.join(fe.select(\'eid\'), on=\'eid\', how=\'semi\')\n vr = TR.join(ve.select(\'eid\'), on=\'eid\', how=\'semi\')\n return fe, fr, ve, vr\n\ndef train_fold_cutoffs(C_val: int):\n """Daily cutoffs strictly before C_val used to build model-training rows (each fold: first event/user in [Ck, Ck+1d))."""\n return [J20 + k * DAY for k in range(1, (C_val - J20) // DAY)]\n', 'v3feat_ne.py': '"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR.\n\nbuild(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate.\nEvery statistic uses ONLY:\n * train events/rows with ts < C (fit window: impressions + labels), and\n * click histories of events with ts < C, plus the target event\'s OWN history (strictly before its ts).\nSince target events are each user\'s first event at/after C, their histories contain only clicks < C,\nso nothing observed after a target row\'s own event time is ever joined.\n\nFeature groups (column prefixes, used for ablation):\n hist_ candidate vs the user\'s click log / own history / past impressions of that user-item pair\n it_ item popularity & CTR (full fit window, last 24h, last 6h, trend)\n co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user\'s last 5/20 clicks\n us_ user activity / CTR / recency / cold-start flags\n ev_ within-event rank / z-score / event max & mean of the main features\n tm_ time_bucket, day_type, hour, staleness since cutoff\n sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines\n mf_ logistic matrix factorisation score fitted on fit-window impressions\n"""\nimport time\nimport numpy as np\nimport polars as pl\nfrom scipy import sparse\nfrom sklearn.utils.extmath import randomized_svd\n\nJ20 = 1737331200\nDAY = 86400\nG_PRIOR = 0.0629\nK_ITEM = 20 # pseudo-impressions for item CTR shrinkage\nK_USER = 5 # pseudo-events for user CTR shrinkage\nTOPK = 200 # co-click neighbours kept per item\nID_COLS = [\'rowid\', \'eid\', \'u\', \'ii\', \'ts\', \'clicked\', \'ord\']\n_t0 = time.time()\n\ndef log(*a):\n print(time.strftime(\'%H:%M:%S\'), *a, flush=True)\n\ndef load(D):\n TE = pl.read_parquet(f\'{D}/ev_train.parquet\')\n TR = pl.read_parquet(f\'{D}/rows_train.parquet\')\n EE = pl.read_parquet(f\'{D}/ev_eval.parquet\')\n ER = pl.read_parquet(f\'{D}/rows_eval.parquet\')\n NI = pl.read_parquet(f\'{D}/items.parquet\').height\n NU = pl.read_parquet(f\'{D}/users.parquet\').height\n return TE, TR, EE, ER, NI, NU\n\ndef _hist_long(E):\n return (E.select(\'u\', \'h\', \'h_ts\').explode([\'h\', \'h_ts\']).drop_nulls(\'h\')\n .select(\'u\', pl.col(\'h\').cast(pl.Int32).alias(\'ii\'), pl.col(\'h_ts\').cast(pl.Int64).alias(\'ts\')))\n\ndef click_log(C, P_ev, P_rows, T_ev):\n """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories."""\n win = P_rows.filter(pl.col(\'clicked\') == 1).select(\'u\', pl.col(\'ii\').cast(pl.Int32), \'ts\')\n hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col(\'ts\') < C)\n CL = pl.concat([win, hist]).unique().sort(\'u\', \'ts\', \'ii\')\n return CL\n\ndef cooc_tables(CL, NI):\n """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next."""\n us = CL.select(\'u\', \'ii\').unique()\n pop = us.group_by(\'ii\').agg(pl.len().alias(\'pop\'))\n pairs = (us.join(us, on=\'u\', suffix=\'_b\').filter(pl.col(\'ii\') != pl.col(\'ii_b\'))\n .group_by(\'ii\', \'ii_b\').agg(pl.len().cast(pl.Int32).alias(\'co\')))\n co = (pairs.join(pop, on=\'ii\').join(pop.rename({\'ii\': \'ii_b\', \'pop\': \'pop_b\'}), on=\'ii_b\')\n .with_columns((pl.col(\'co\') / (pl.col(\'pop\') * pl.col(\'pop_b\')).sqrt()).cast(pl.Float32).alias(\'cos\'))\n .select(pl.col(\'ii_b\').alias(\'j\'), \'ii\', \'cos\') # j = history item, ii = candidate\n .sort(\'cos\', descending=True).group_by(\'j\').head(TOPK))\n seq = CL.select(\'u\', \'ii\') # already sorted by u, ts\n parts = []\n for lag in (1, 2, 3):\n parts.append(seq.with_columns(pl.col(\'ii\').shift(lag).over(\'u\').alias(\'j\')).drop_nulls(\'j\')\n .select(pl.col(\'j\').cast(pl.Int32), \'ii\', pl.lit(1.0 / lag).alias(\'w\')))\n trn = pl.concat(parts).filter(pl.col(\'j\') != pl.col(\'ii\')).group_by(\'j\', \'ii\').agg(pl.col(\'w\').sum())\n tot = trn.group_by(\'j\').agg(pl.col(\'w\').sum().alias(\'tot\'))\n trn = (trn.join(tot, on=\'j\').with_columns((pl.col(\'w\') / (pl.col(\'tot\') + 5.0)).cast(pl.Float32).alias(\'trp\'))\n .select(\'j\', \'ii\', \'trp\').sort(\'trp\', descending=True).group_by(\'j\').head(TOPK))\n return co, trn\n\ndef _segmax(n, idx, vals, init=-1.0):\n """v8: == np.maximum.at(np.full(n, init), idx, vals), via sort + reduceat."""\n out = np.full(n, init, np.float32)\n if len(idx) == 0: return out\n o = np.argsort(idx, kind=\'stable\'); si = idx[o]; sv = vals[o].astype(np.float32)\n st = np.flatnonzero(np.r_[True, si[1:] != si[:-1]])\n out[si[st]] = np.maximum(out[si[st]], np.maximum.reduceat(sv, st)); return out\n\ndef fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000):\n """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy)."""\n rng = np.random.default_rng(seed)\n u = P_rows[\'u\'].to_numpy().astype(np.int64); i = P_rows[\'ii\'].to_numpy().astype(np.int64)\n y = P_rows[\'clicked\'].to_numpy().astype(np.float32)\n P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32)\n bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32)\n b = np.float32(np.log(y.mean() / (1 - y.mean())))\n gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6)\n n = len(y)\n for ep in range(epochs):\n perm = rng.permutation(n)\n for s in range(0, n, batch):\n idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx]\n pu, qi = P[ub], Q[ib]\n z = b + bu[ub] + bi[ib] + (pu * qi).sum(1)\n g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz\n m = len(idx)\n Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m))\n Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m))\n dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None]\n dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None]\n dbu = Su @ g; dbi = Si @ g\n gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi\n P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ)\n bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi)\n return dict(b=b, bu=bu, bi=bi, P=P, Q=Q)\n\ndef build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode=\'std\'):\n t0 = time.time()\n P_ev = TE.filter(pl.col(\'ts\') < C)\n P_rows = TR.join(P_ev.select(\'eid\', \'u\', \'ts\'), on=\'eid\', how=\'inner\')\n g = float(P_rows[\'clicked\'].mean())\n ndays = (C - J20) / DAY\n CL = click_log(C, P_ev, P_rows, T_ev) # per-user stats: train data + each target row\'s OWN history\n CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # item-level / global stats: train data only (no target-event histories)\n\n # ---------------- item ----------------\n def ictr(c, n):\n return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM)\n it = P_rows.group_by(\'ii\').agg(\n pl.len().alias(\'it_imp\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'it_clk\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'it_imp24\'), pl.col(\'clicked\').filter(pl.col(\'ts\') >= C - DAY).sum().cast(pl.Int32).alias(\'it_clk24\'),\n (pl.col(\'ts\') >= C - 6 * 3600).sum().alias(\'it_imp6\'), pl.col(\'clicked\').filter(pl.col(\'ts\') >= C - 6 * 3600).sum().cast(pl.Int32).alias(\'it_clk6\'))\n it = it.with_columns(ictr(\'it_clk\', \'it_imp\').alias(\'it_ctr\'), ictr(\'it_clk24\', \'it_imp24\').alias(\'it_ctr24\'),\n ictr(\'it_clk6\', \'it_imp6\').alias(\'it_ctr6\'))\n it = it.with_columns((pl.col(\'it_ctr24\') / pl.col(\'it_ctr\')).alias(\'it_trend24\'), (pl.col(\'it_ctr6\') / pl.col(\'it_ctr\')).alias(\'it_trend6\'),\n (pl.col(\'it_imp24\') / (pl.col(\'it_imp\') / ndays)).alias(\'it_imptrend24\'),\n (pl.col(\'it_imp\') / ndays).alias(\'it_imp_rate\'))\n ip = CLg.group_by(\'ii\').agg(pl.len().alias(\'it_nclk_all\'), pl.col(\'u\').n_unique().alias(\'it_nusers\'),\n (pl.col(\'ts\') < J20).sum().alias(\'it_nclk_warm\'),\n ((pl.col(\'ts\') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias(\'it_clk_dec2d\'))\n\n # ---------------- user ----------------\n ue = P_rows.group_by(\'u\').agg(pl.col(\'eid\').n_unique().alias(\'us_nev\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'us_nclk_win\'),\n pl.col(\'ts\').max().alias(\'_u_last_ev\'))\n ue = ue.with_columns(((pl.col(\'us_nclk_win\') + K_USER * 0.629) / (pl.col(\'us_nev\') + K_USER)).alias(\'us_ctr\'))\n uc = CL.group_by(\'u\').agg(pl.len().alias(\'us_nclk_all\'), (pl.col(\'ts\') < J20).sum().alias(\'us_nclk_warm\'),\n pl.col(\'ii\').n_unique().alias(\'us_nuniq\'), pl.col(\'ts\').max().alias(\'_u_last_clk\'),\n (pl.col(\'ts\') >= C - DAY).sum().alias(\'us_nclk24\'))\n uc = uc.with_columns((1 - pl.col(\'us_nuniq\') / pl.col(\'us_nclk_all\')).alias(\'us_repeat_rate\'))\n\n # ---------------- user x item ----------------\n uic = CL.group_by(\'u\', \'ii\').agg(pl.len().alias(\'hist_n_all\'), (pl.col(\'ts\') < J20).sum().alias(\'hist_n_warm\'),\n (pl.col(\'ts\') >= J20).sum().alias(\'hist_n_win\'), pl.col(\'ts\').max().alias(\'_ui_last_clk\'))\n uii = P_rows.group_by(\'u\', \'ii\').agg(pl.len().alias(\'hist_imp\'), pl.col(\'clicked\').sum().cast(pl.Int32).alias(\'_ui_clk\'),\n pl.col(\'ts\').max().alias(\'_ui_last_imp\'))\n uii = uii.with_columns((pl.col(\'hist_imp\') - pl.col(\'_ui_clk\')).alias(\'hist_noclk\'))\n\n # ---------------- assemble ----------------\n R = (T_rows.select([c for c in (\'rowid\', \'eid\', \'ii\', \'clicked\', \'ord\') if c in T_rows.columns])\n .join(T_ev.select(\'eid\', \'u\', \'ts\', \'tb\', \'dt\', \'h\', \'h_ts\'), on=\'eid\', how=\'left\').with_row_index(\'r\'))\n R = (R.join(it, on=\'ii\', how=\'left\').join(ip, on=\'ii\', how=\'left\').join(ue, on=\'u\', how=\'left\').join(uc, on=\'u\', how=\'left\')\n .join(uic, on=[\'u\', \'ii\'], how=\'left\').join(uii, on=[\'u\', \'ii\'], how=\'left\'))\n R = R.with_columns(\n pl.col(\'h\').list.len().alias(\'us_hlen\'),\n (pl.col(\'h\').list.len() == 0).cast(pl.Int8).alias(\'us_nohist\'),\n pl.col(\'us_nev\').is_null().cast(pl.Int8).alias(\'us_cold\'),\n (pl.col(\'ts\') - pl.col(\'_u_last_ev\')).alias(\'us_since_ev\'),\n (pl.col(\'ts\') - pl.col(\'_u_last_clk\')).alias(\'us_since_clk\'),\n (pl.col(\'ts\') - pl.col(\'_ui_last_clk\')).alias(\'hist_since_clk\'),\n (pl.col(\'ts\') - pl.col(\'_ui_last_imp\')).alias(\'hist_since_imp\'),\n (pl.col(\'ts\') - C).alias(\'tm_stale\'), ((pl.col(\'ts\') // 3600) % 24).cast(pl.Int8).alias(\'tm_hour\'),\n pl.col(\'tb\').alias(\'tm_tb\'), pl.col(\'dt\').alias(\'tm_dt\'))\n # position (from end, 1 = most recent) of candidate in own history\n hx = (R.select(\'r\', \'ii\', \'h\').with_columns(pl.int_ranges(pl.col(\'h\').list.len(), 0, -1).alias(\'pos\'))\n .explode([\'h\', \'pos\']).drop_nulls(\'h\').filter(pl.col(\'h\') == pl.col(\'ii\'))\n .group_by(\'r\').agg(pl.col(\'pos\').min().alias(\'hist_pos\'), pl.len().alias(\'hist_n_own\')))\n R = R.join(hx, on=\'r\', how=\'left\')\n R = R.with_columns(pl.col(\'hist_since_clk\').fill_null(30 * DAY), pl.col(\'hist_since_imp\').fill_null(30 * DAY),\n pl.col(\'hist_pos\').fill_null(99), pl.col(\'hist_n_own\').fill_null(0), pl.col(\'us_since_ev\').fill_null(30 * DAY), pl.col(\'us_since_clk\').fill_null(30 * DAY))\n R = R.with_columns((pl.col(\'hist_n_all\').fill_null(0) > 0).cast(pl.Int8).alias(\'hist_in\'))\n\n # ---------------- co-click ----------------\n co, trn = cooc_tables(CLg, NI)\n last20 = CL.group_by(\'u\', maintain_order=True).agg(pl.col(\'ii\').tail(20).alias(\'l20\'))\n T_u = R.select(\'r\', \'u\', \'ii\').join(last20, on=\'u\', how=\'left\')\n X = (T_u.with_columns(pl.int_ranges(pl.col(\'l20\').list.len(), 0, -1).alias(\'pos\')).explode([\'l20\', \'pos\'])\n .drop_nulls(\'l20\').rename({\'l20\': \'j\'}).with_columns(pl.col(\'j\').cast(pl.Int32))\n .join(co, on=[\'j\', \'ii\'], how=\'left\').join(trn, on=[\'j\', \'ii\'], how=\'left\')\n .with_columns(pl.col(\'cos\').fill_null(0), pl.col(\'trp\').fill_null(0)))\n l5 = pl.col(\'pos\') <= 5\n agg = X.group_by(\'r\').agg(\n pl.col(\'cos\').max().alias(\'co_cos_max20\'), pl.col(\'cos\').mean().alias(\'co_cos_mean20\'),\n pl.col(\'cos\').filter(l5).max().alias(\'co_cos_max5\'), pl.col(\'cos\').filter(l5).mean().alias(\'co_cos_mean5\'),\n pl.col(\'cos\').filter(pl.col(\'pos\') == 1).first().alias(\'co_cos_last\'),\n pl.col(\'trp\').max().alias(\'co_tr_max20\'), pl.col(\'trp\').mean().alias(\'co_tr_mean20\'),\n pl.col(\'trp\').filter(l5).max().alias(\'co_tr_max5\'), pl.col(\'trp\').filter(l5).mean().alias(\'co_tr_mean5\'),\n pl.col(\'trp\').filter(pl.col(\'pos\') == 1).first().alias(\'co_tr_last\'),\n (pl.col(\'cos\') / pl.col(\'pos\').cast(pl.Float32).sqrt()).sum().alias(\'co_cos_rw\'), pl.col(\'cos\').sum().alias(\'co_cos_sum20\'))\n R = R.join(agg, on=\'r\', how=\'left\').sort(\'r\')\n\n # ---------------- SVD embeddings ----------------\n # sv_mode (leakage checks only): \'std\' = click log as of C; \'notgt\' = drop target events\' own histories;\n # \'lag1d\' = only clicks/impressions with ts < C - 1 day\n if sv_mode == \'std\':\n CLsv, NEGsrc = CLg, P_rows\n elif sv_mode == \'notgt\':\n CLsv = pl.concat([P_rows.filter(pl.col(\'clicked\') == 1).select(\'u\', pl.col(\'ii\').cast(pl.Int32), \'ts\'),\n _hist_long(P_ev).filter(pl.col(\'ts\') < C)]).unique()\n NEGsrc = P_rows\n elif sv_mode == \'lag1d\':\n CLsv, NEGsrc = CL.filter(pl.col(\'ts\') < C - DAY), P_rows.filter(pl.col(\'ts\') < C - DAY)\n else:\n raise ValueError(sv_mode)\n assert (CLsv[\'ts\'].max() or 0) < C and (NEGsrc[\'ts\'].max() or 0) < C\n cnt = CLsv.group_by(\'u\', \'ii\').agg(pl.len().alias(\'n\'))\n Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy())), shape=(NU, NI))\n df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)\n Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc\n # user-side matrix incl. each target user\'s own history (fold-in onto item factors learned from train-only clicks)\n cnt = CL.filter(pl.col(\'ts\') < C).group_by(\'u\', \'ii\').agg(pl.len().alias(\'n\')) if sv_mode == \'std\' else cnt\n Xu = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy())), shape=(NU, NI)) @ sparse.diags(idf)\n Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu\n uu = R[\'u\'].to_numpy().astype(np.int64); iic = R[\'ii\'].to_numpy().astype(np.int64)\n ex = X.select(\'r\', \'j\', \'pos\').unique([\'r\', \'j\', \'pos\'])\n # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway\n er_r = ex[\'r\'].to_numpy(); er_j = ex[\'j\'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex[\'pos\'].to_numpy())).astype(np.float32)\n er_l5 = ex[\'pos\'].to_numpy() <= 5\n nR = R.height\n svcols = {}\n Werj = sparse.csr_matrix((er_w, (er_r, er_j)), shape=(nR, NI))\n for k in (16, 32, 64):\n U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)\n V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = np.asarray(Xu @ (Vt.T / np.sqrt(S))).astype(np.float32) # fold-in = U*sqrt(S) for train users\n Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9)\n cand = Vn[iic]\n svcols[f\'sv_k{k}_ucos\'] = (Un[uu] * cand).sum(1)\n svcols[f\'sv_k{k}_udot\'] = (Uf[uu] * V[iic]).sum(1)\n hv = np.asarray(Werj @ Vn, dtype=np.float32) # v8: sparse matmul replaces np.add.at\n hn = np.linalg.norm(hv, axis=1); svcols[f\'sv_k{k}_hcos\'] = (hv * cand).sum(1) / (hn + 1e-9)\n if k == 32:\n svcols[\'sv_k32_hnorm\'] = hn\n sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1)\n mx = _segmax(nR, er_r, sims); mx5 = _segmax(nR, er_r[er_l5], sims[er_l5])\n svcols[\'sv_k32_imax20\'] = mx; svcols[\'sv_k32_imax5\'] = mx5\n if k == 64:\n # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only)\n Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt[\'u\'].to_numpy(), cnt[\'ii\'].to_numpy())), shape=(NU, NI))\n neg = NEGsrc.filter(pl.col(\'clicked\') == 0)\n Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg[\'u\'].to_numpy(), neg[\'ii\'].to_numpy())), shape=(NU, NI))\n nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9\n nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9\n cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1)\n svcols[\'sv_k64_ccos\'] = cc; svcols[\'sv_k64_ncos\'] = cn\n for a, nm in ((0.25, \'025\'), (0.5, \'05\'), (1.0, \'1\')):\n M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9\n svcols[f\'sv_k64_con{nm}\'] = (M[uu] * cand).sum(1)\n R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()])\n\n # ---------------- MF ----------------\n if with_mf:\n mf = fit_mf(P_rows, NU, NI)\n uu = R[\'u\'].to_numpy().astype(np.int64); ii = R[\'ii\'].to_numpy().astype(np.int64)\n dot = (mf[\'P\'][uu] * mf[\'Q\'][ii]).sum(1)\n R = R.with_columns(pl.Series(\'mf_score\', mf[\'b\'] + mf[\'bu\'][uu] + mf[\'bi\'][ii] + dot),\n pl.Series(\'mf_dot\', dot), pl.Series(\'mf_bi\', mf[\'bi\'][ii]))\n\n fill0 = [c for c in R.columns if c.startswith((\'it_imp\', \'it_clk\', \'it_n\', \'us_n\', \'hist_n\', \'hist_imp\', \'hist_noclk\', \'co_\')) and R[c].dtype != pl.List]\n R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])\n R = R.with_columns([pl.col(c).fill_null(g) for c in (\'it_ctr\', \'it_ctr24\', \'it_ctr6\')] +\n [pl.col(c).fill_null(1.0) for c in (\'it_trend24\', \'it_trend6\', \'it_imptrend24\')] +\n [pl.col(\'us_ctr\').fill_null(g), pl.col(\'it_imp_rate\').fill_null(0), pl.col(\'us_repeat_rate\').fill_null(0)])\n\n # ---------------- event context ----------------\n main = [\'it_ctr\', \'it_ctr24\', \'it_imp\', \'it_nclk_all\', \'it_clk_dec2d\', \'hist_n_all\', \'hist_noclk\',\n \'co_cos_max20\', \'co_cos_rw\', \'co_tr_max5\', \'co_tr_last\',\n \'sv_k16_ucos\', \'sv_k32_ucos\', \'sv_k64_ucos\', \'sv_k32_hcos\', \'sv_k64_hcos\', \'sv_k32_imax20\', \'sv_k64_con05\', \'sv_k64_con025\', \'sv_k64_ccos\'] + ([\'mf_score\', \'mf_dot\'] if with_mf else [])\n exprs = []\n for c in main:\n x = pl.col(c).cast(pl.Float64)\n exprs += [x.rank(\'average\').over(\'eid\').alias(f\'ev_{c}_rk\'),\n ((x - x.mean().over(\'eid\')) / (x.std().over(\'eid\') + 1e-9)).alias(f\'ev_{c}_z\'),\n x.max().over(\'eid\').alias(f\'ev_{c}_max\'), x.mean().over(\'eid\').alias(f\'ev_{c}_mean\')]\n R = R.with_columns(exprs)\n R = R.drop([c for c in R.columns if c.startswith(\'_\')] + [\'h\', \'h_ts\', \'tb\', \'dt\']).sort(\'r\').drop(\'r\')\n log(f\'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s\')\n return R\n\nGROUPS = [\'it\', \'hist\', \'us\', \'co\', \'ev\', \'tm\', \'sv\', \'mf\']\n\ndef feature_cols(df, groups):\n cols = []\n for c in df.columns:\n if c in ID_COLS: continue\n gname = c.split(\'_\')[0]\n if gname == \'ev\':\n src = c[3:].split(\'_\')[0] # ev__...\n if \'ev\' in groups and src in groups: cols.append(c)\n elif gname in groups:\n cols.append(c)\n return cols\n', 'v3fold.py': '"""v3 features (v2work/v3feat_ne.py = v3/features.py with global/item stats from train data only) for all fold specs.\nUsage: python v3fold.py NAME... -> v2work/feat_v3/NAME.parquet\n f1..f6: first event/user in [Ck, Ck+1d); val25/val26: [C, end of train); eval: Jan27 cutoff\n e1..e4: [Ck, Jan25) (eval-like staleness, for cutoff-25 validation); E1..E6: [Ck, Jan27) (eval-like, final fit)"""\nimport sys, os\nsys.path.insert(0, \'@@WORK@@\'); sys.path.insert(1, \'@@WORK@@/v3\')\nfrom v3feat_ne import load, build, log\nfrom split import J20, DAY, first_events\nTE, TR, EE, ER, NI, NU = load(\'@@WORK@@/v3/data\')\nOUT = \'@@WORK@@/feat_v3\'; os.makedirs(OUT, exist_ok=True)\nSPEC = {**{f\'f{k}\': (J20 + k * DAY, DAY) for k in range(1, 7)}, \'val25\': (J20 + 5 * DAY, None), \'val26\': (J20 + 6 * DAY, None),\n **{f\'e{k}\': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f\'E{k}\': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f\'d{k}\': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}}\nfor n in sys.argv[1:]:\n if n == \'eval\': R = build(J20 + 7 * DAY, EE, ER, TE, TR, NI, NU)\n else:\n C, W = SPEC[n]; T_ev = first_events(TE, C, W)\n R = build(C, T_ev, TR.join(T_ev.select(\'eid\'), on=\'eid\', how=\'semi\'), TE, TR, NI, NU)\n R.write_parquet(f\'{OUT}/{n}.parquet\'); log(\'wrote\', n, R.shape)\n', 'validate.py': '"""Standalone validator: python validate.py SUB.csv"""\nimport sys, csv, math, numpy as np, polars as pl\nout=sys.argv[1]\nids=pl.read_csv(\'@@WORK@@/evaluation.csv\',columns=[\'rowid\'],schema_overrides={\'rowid\':pl.Utf8})[\'rowid\'].to_list()\nraw=open(out,\'rb\').read(); errs=[]\ntry: raw.decode(\'utf-8\')\nexcept Exception: errs.append(\'not utf-8\')\nwith open(out,newline=\'\',encoding=\'utf-8\') as f:\n r=csv.reader(f); h=next(r); rows=list(r)\nif h!=[\'rowid\',\'predicted_probability\']: errs.append(f\'header {h}\')\nif len(rows)!=529390: errs.append(f\'rows {len(rows)}\')\nsid=[x[0] for x in rows]\nif len(set(sid))!=len(sid): errs.append(\'dup rowids\')\nif set(sid)!=set(ids): errs.append(\'rowid set mismatch\')\nv=np.array([float(x[1]) for x in rows])\nif not (np.isfinite(v).all() and (v>=0).all() and (v<=1).all()): errs.append(\'bad values\')\nprint(f\'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.5g} max={v.max():.4g} distinct={len(np.unique(v))}\')\nprint(\'VALID\' if not errs else \'INVALID \'+\';\'.join(errs))\n', 'write_sub.py': '"""Write + validate submission from eval_pred.parquet. Usage: python write_sub.py OUT.csv"""\nimport sys, csv, math, hashlib, zipfile, os, numpy as np, polars as pl\nout=sys.argv[1]\npr=pl.read_parquet(os.environ.get(\'PRED\',\'@@WORK@@/eval_pred.parquet\'))\nev=pl.read_csv(\'@@WORK@@/evaluation.csv\',columns=[\'rowid\'],schema_overrides={\'rowid\':pl.Utf8})\ndf=ev.join(pr,on=\'rowid\',how=\'left\',maintain_order=\'left\')\np=np.nan_to_num(df[\'p\'].to_numpy().astype(np.float64),nan=0.0629,posinf=1.0,neginf=0.0).clip(0,1)\np[np.isnan(df[\'p\'].to_numpy().astype(float))]=0.0629\nprint(\'missing preds filled:\',int(df[\'p\'].null_count()))\nwith open(out,\'w\',encoding=\'utf-8\',newline=\'\') as f:\n f.write(\'rowid,predicted_probability\\n\'); f.writelines(f\'{r},{v:.8g}\\n\' for r,v in zip(df[\'rowid\'].to_list(),p))\n# ---- validator\nids=ev[\'rowid\'].to_list(); errs=[]\nwith open(out,newline=\'\',encoding=\'utf-8\') as f:\n r=csv.reader(f); h=next(r); rows=list(r)\nif h!=[\'rowid\',\'predicted_probability\']: errs.append(\'header\')\nif len(rows)!=529390 or len(ids)!=529390: errs.append(f\'rows {len(rows)}\')\nsid=[x[0] for x in rows]\nif len(set(sid))!=len(sid) or set(sid)!=set(ids): errs.append(\'rowid set mismatch\')\nvals=[float(x[1]) for x in rows]\nif not all(math.isfinite(v) and 0<=v<=1 for v in vals): errs.append(\'bad values\')\nv=np.array(vals); print(f\'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.4g} max={v.max():.4g} distinct={len(np.unique(v))}\')\nprint(\'VALID\' if not errs else \'INVALID \'+\';\'.join(errs))\nzp=out+\'.zip\'\nwith zipfile.ZipFile(zp,\'w\',zipfile.ZIP_DEFLATED) as z: z.write(out,arcname=os.path.basename(out))\nwith zipfile.ZipFile(zp) as z: print(\'zip contents\',z.namelist())\nprint(\'sha256\',hashlib.sha256(open(out,\'rb\').read()).hexdigest())\n'} + +SLOW = '"learning_rate":0.03,"feature_fraction":0.55,"num_leaves":95,"lambda_l2":30' +BIG = '"learning_rate":0.015,"num_leaves":127,"feature_fraction":0.5,"lambda_l2":30,"min_data_in_leaf":200,"max_bin":255,"force_col_wise":true' +LP = '"max_bin":63,"force_col_wise":true' +LR = 'feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw' +DROP = "'^sq_(pdot|pcos|mdot|mcos)$'" +def pipeline(W, RAW, T, PY, OUT, cut): + """ordered (stage, shell command or None for setup, subdir of W)""" + FOLDS = 'f3 f4 f5 f6 eval' if cut == 27 else 'f1 f2 f3 f4 val25' + V2 = 'lite' if cut == 27 else 'c25' + return [('setup', None, ''), ('prep_v3', f'{PY} prep.py {RAW}', 'v3'), ('prep_v2', f'{PY} prep.py', ''), + ('feat_v3', f'{PY} v3fold.py {FOLDS}', ''), ('feat_v2', f'{PY} build_folds.py {V2}', ''), + ('feat_x', f'{PY} extra.py {FOLDS}', ''), ('feat_lr', f'LR_KS=4,6,8,10,12,16 LR_CON=1 {PY} lrfeat2.py {FOLDS}', ''), + ('feat_tt', f'TT_THREADS={T} {PY} ttfeat.py {FOLDS}', ''), ('feat_pp', f'{PY} ppmi.py {FOLDS}', ''), + ('feat_sq', f'V12_THREADS={T} {PY} seqrun.py {FOLDS}', ''), ('feat_dw', f'{PY} dw/mkdw.py {FOLDS}', ''), + ('dwbig', f"{PY} ens.py {cut} dwbig --seeds 0 --rounds 1000 --save 1000 --threads {T} --lr {LR} --nfolds 4 --drop {DROP} --params '{{{BIG}}}'", ''), + ('dw_s', f"{PY} ens.py {cut} dw_s --seeds 0 --rounds 500 --save 500 --threads {T} --lr {LR} --nfolds 4 --negfrac 0.5 --drop {DROP} --params '{{{SLOW},{LP}}}'", ''), + *[(f'mlp{s}', f'MLP_THREADS={T} {PY} mlp12.py {cut} dwmlp --ep 4 --seed {s} --lr {LR}', '') for s in range(3)], + ('blend', f'{PY} blend12.py {cut}', ''), + *([('write', f'PRED={W}/eval_pred.parquet {PY} write_sub.py {OUT}', ''), ('validate', f'{PY} validate.py {OUT}', '')] if cut == 27 else [])] +def setup(W, RAW): + os.makedirs(os.path.join(W, 'v3'), exist_ok=True); os.makedirs(os.path.join(W, 'ens'), exist_ok=True) + for k, s in SRC.items(): + os.makedirs(os.path.dirname(os.path.join(W, k)), exist_ok=True) + open(os.path.join(W, k), 'w').write(s.replace('@@WORK@@', W).replace('@@RAW@@', RAW)) + for f in ('train.csv', 'evaluation.csv'): # v2 prep / writer read the raw CSVs from WORK + dst = os.path.join(W, f) + if not os.path.exists(dst): os.symlink(os.path.join(RAW, f), dst) +def thread_env(T): + return dict(os.environ, POLARS_MAX_THREADS=T, OMP_NUM_THREADS=T, OPENBLAS_NUM_THREADS=T, MKL_NUM_THREADS=T, NUMEXPR_NUM_THREADS=T, + TT_THREADS=T, SQ_THREADS=T, RAYON_NUM_THREADS=T, PYTHONUNBUFFERED='1') +def main(): + ap = argparse.ArgumentParser(); ap.add_argument('--raw', required=True); ap.add_argument('--work', default='./v12_work') + ap.add_argument('--threads', type=int, default=8); ap.add_argument('--out', default=None); ap.add_argument('--from', dest='frm', default='setup'); ap.add_argument('--to', default=None) + ap.add_argument('--cut', type=int, default=27, choices=(25, 27)) + a = ap.parse_args() + W = os.path.abspath(a.work); RAW = os.path.abspath(a.raw); T = str(a.threads) + OUT = os.path.abspath(a.out or os.path.join(W, 'submission_v12.csv')) + os.makedirs(os.path.dirname(OUT), exist_ok=True) + env = thread_env(T); PY = sys.executable + P = pipeline(W, RAW, T, PY, OUT, a.cut); names = [p[0] for p in P]; to = a.to or names[-1] + tf = os.path.join(W, 'timings.json'); tim = json.load(open(tf)) if os.path.exists(tf) and a.frm != 'setup' else {} + t_all = time.time() + for st, cmd, sub in P[names.index(a.frm):names.index(to) + 1]: + t0 = time.time() + if cmd is None: setup(W, RAW) + else: print('>>', cmd, flush=True); subprocess.run(cmd, shell=True, check=True, cwd=os.path.join(W, sub), env=env) + tim[st] = round(time.time() - t0, 1); print(f'[stage {st}] {tim[st]:.0f}s', flush=True) + os.makedirs(W, exist_ok=True); json.dump(tim, open(tf, 'w'), indent=1) + print('TOTAL this run %.1f min; all stages %.1f min' % ((time.time() - t_all) / 60, sum(tim.values()) / 60)) + if a.cut == 27: print('submission:', OUT) +if __name__ == '__main__': + main() diff --git a/v12/sol_work/cmp_base.py b/v12/sol_work/cmp_base.py new file mode 100644 index 0000000000000000000000000000000000000000..2a39ab38162a91ac39df6500834209dfe2c49a0f --- /dev/null +++ b/v12/sol_work/cmp_base.py @@ -0,0 +1,13 @@ +import sys, json, numpy as np, polars as pl +W=sys.argv[1]; sys.path.insert(0,W); import os; os.chdir(W) +from cl import matrix, BASE +X,m,fl=matrix('eval',1.0,extra=BASE) +ref='/workspace/huawei/v9d/val/cache_mx/eval_93841df7c9f9.parquet'; rfl=json.load(open(ref+'.json')) +print('ncols sol',len(fl),'ref',len(rfl),'same list',fl==rfl, 'same set',set(fl)==set(rfl)) +r=pl.read_parquet(ref); print('rows',len(r),m.height,'rowid same order',(r['rowid'].to_numpy()==m['rowid'].to_numpy()).all()) +R=r.select(rfl).to_numpy().astype(np.float32); idx=[fl.index(c) for c in rfl] if set(fl)==set(rfl) else None +Xs=X[:,idx]; nanm=np.isnan(R)!=np.isnan(Xs); print('nan-pattern mismatches cols',int((nanm.sum(0)>0).sum())) +d=np.nanmax(np.abs(R-Xs),0); sc=np.nanmax(np.abs(R),0)+1e-9; rel=d/sc +bad=np.argsort(-rel)[:12]; print('worst rel diffs',[(rfl[i],float(rel[i])) for i in bad]); print('cols with rel>1e-4:',int((rel>1e-4).sum()),'rel>1e-2',int((rel>1e-2).sum())) +fr=np.nanmean(np.abs(R-Xs)>1e-3*(np.abs(R)+1e-3),0); print('cols with >0.1% of values differing (rel 1e-3):',int((fr>1e-3).sum()),'; max frac',float(np.nanmax(fr))) +for i in np.argsort(-fr)[:10]: print(rfl[i],round(float(fr[i]),5), 'corr',round(float(np.corrcoef(np.nan_to_num(R[:,i]),np.nan_to_num(Xs[:,i]))[0,1]),6)) diff --git a/v12/sol_work/head12.txt b/v12/sol_work/head12.txt new file mode 100644 index 0000000000000000000000000000000000000000..218987fdce7e84d9a114e6bff041418e57a085c0 --- /dev/null +++ b/v12/sol_work/head12.txt @@ -0,0 +1,23 @@ +"""Huawei TechArena 2026 Phase 1 - submission_v12: ONE self-contained end-to-end script (raw CSVs -> features -> models -> submission CSV). + +v12 recipe (local AUC: C25 0.81723 / C26 0.82604 / C23 0.79566; v10 was 0.8157 / 0.8247 / 0.7938 at C25 / C26 / C23): + features v8/v10 feature families (v3 + v2 + extra similarity + feat_lr2 + feat_tt + feat_pp; 'notime') -> 377 columns + + feat_sqC : DIN-style sequence model (seq11.py), refit at every fold cutoff, averaged over 11 seeds (0-10; seeds 8-10 use 4 heads/hid 128) + -> 20 stacking columns (logit, within-event z/dm/rank/gaps, attention stats, cosines, seed sd); 4 of them dropped (ens.py --drop) + + feat_dw : DW impression-taste (SVD of the user x item IMPRESSION matrix), slate-centrality and skip-contrast features -> 42 columns + = 435 model columns. All aggregates use train events with ts < fit cutoff only; each eval row only uses its own history and own slate. + models dwbig : LightGBM lr .015, 127 leaves, ff .5, l2 30, min_data 200, max_bin 255, negatives kept 100%, 1000 rounds, folds f3..f6 + dw_s : LightGBM lr .03, 95 leaves, ff .55, l2 30, max_bin 63, negatives kept 50% (seed 123), 500 rounds, folds f3..f6 + dwmlp : 512-256 MLP (rank-gauss inputs, 4 epochs, OneCycle), 3 seeds averaged, folds f3..f6 (negatives kept 50%) + blend z = (dwbig + dw_s + 2 * mean(dwmlp)) / 4 (logit blend 1:1:2), p = sigmoid(z) in [0, 1] + folds f_k = first event per user in [Jan20 + k d, + 1 d), k = 3..6 (labels known), stats frozen at that cutoff; eval features at Jan27. + +Usage: python solution_v12.py --raw DIR_WITH_train.csv_AND_evaluation.csv [--work WORKDIR] [--threads 8] [--out submission_v12.csv] + [--from STAGE] [--to STAGE] [--cut 27|25] + --cut 25 : local-validation mode (same recipe, folds f1..f4, scored on val25 = first event per user in [Jan25, Jan27) with the train labels; prints AUC). +Env knobs: V12_SQ_SEEDS (sequence seeds, default 11; e.g. 3 for a faster, slightly weaker run), V12_SQ_PAR (concurrent seq runs, default min(3, threads//2)), + V12_SQ_THREADS (torch threads per seq run), V12_MLP_SEEDS is fixed at 3. +Stages (resumable with --from/--to): setup prep_v3 prep_v2 feat_v3 feat_v2 feat_x feat_lr feat_tt feat_pp feat_sq feat_dw dwbig dw_s mlp0 mlp1 mlp2 blend write validate +Needs: python 3.11+ (tested 3.13), see requirements.txt (polars, numpy, scipy, scikit-learn, lightgbm, torch CPU). ~12 GB RAM, ~40 GB disk in WORKDIR. +The module sources are embedded below (SRC) and written to WORKDIR at start-up (no absolute paths anywhere; everything lives under --raw / --work / --out).""" +import os, sys, time, json, argparse, subprocess diff --git a/v12/sol_work/make_sol12.py b/v12/sol_work/make_sol12.py new file mode 100644 index 0000000000000000000000000000000000000000..f4002e4643119cbcbac219cf67543f50b37555d9 --- /dev/null +++ b/v12/sol_work/make_sol12.py @@ -0,0 +1,10 @@ +"""build /workspace/huawei/v12/sol/solution_v12.py from sol_work/src12 (+ head12/tail12).""" +import os +B = '/workspace/huawei/v12/sol_work'; S = f'{B}/src12'; SRC = {} +for r, _, fs in os.walk(S): + for f in sorted(fs): + if f.endswith(('.py', '.sh')) and '__pycache__' not in r: + p = os.path.join(r, f); SRC[os.path.relpath(p, S)] = open(p).read() +head = open(f'{B}/head12.txt').read(); tail = open(f'{B}/tail12.txt').read() +open('/workspace/huawei/v12/sol/solution_v12.py', 'w').write(head + 'SRC = ' + repr(dict(sorted(SRC.items()))) + tail) +print('ok', sorted(SRC)) diff --git a/v12/sol_work/src10/bgseq.sh b/v12/sol_work/src10/bgseq.sh new file mode 100644 index 0000000000000000000000000000000000000000..ea2190c04d6fba16c4bbfe482d01d0e242999ef8 --- /dev/null +++ b/v12/sol_work/src10/bgseq.sh @@ -0,0 +1,4 @@ +# v10: start the sequence-model stage in the background (it only needs v3/data from prep_v3) pinned to the last core with 1 torch thread, while the polars feature stages run on the other cores +rm -f seq10.rc seq10.log +( taskset -c "$2" "$1" seq10.py f3 f4 f5 f6 eval > seq10.log 2>&1; echo $? > seq10.rc ) < /dev/null > /dev/null 2>&1 & +echo "seq10 started in background on core $2 (SQ_THREADS=$SQ_THREADS)" diff --git a/v12/sol_work/src10/blend_lite.py b/v12/sol_work/src10/blend_lite.py new file mode 100644 index 0000000000000000000000000000000000000000..a9dbc9c8872e5a22c56c3841ecc841c435760581 --- /dev/null +++ b/v12/sol_work/src10/blend_lite.py @@ -0,0 +1,20 @@ +"""v10 final blend (no sequence post-blend: the sequence model is stacked into both GBDTs via feat_sq10) +v8 final blend on eval -> eval_pred.parquet (rowid, p). +z = mean of 2 GBDT component logits (base = 1 seed at 400 rounds, slow = 1 seed at 500 rounds); +z8 = 0.8 * z + 0.2 * sq_logit (seq.py sequence model, raw logit), p = sigmoid(z8) in [0, 1].""" +import glob, numpy as np, polars as pl +from cl import matrix +COL = {'base': 1, 'slow': 2}; NS = {'base': 1, 'slow': 1}; WSQ = 0.0 +_, m, _ = matrix('eval', 1.0) +c = {} +for t, ci in COL.items(): + fs = sorted(glob.glob(f'ens/{t}_c27_s*.npy')); assert len(fs) == NS[t], (t, fs) + c[t] = np.mean([np.load(f)[:, ci] for f in fs], 0); assert len(c[t]) == m.height +z = np.mean(list(c.values()), 0) +sq = pl.read_parquet('feat_sq10/eval.parquet', columns=['rowid', 'sq_logit']) +j = m.select('rowid').join(sq.with_columns(pl.col('rowid').cast(m['rowid'].dtype)), on='rowid', how='left', maintain_order='left') +assert j.height == m.height and j['sq_logit'].null_count() == 0 and (j['rowid'] == m['rowid']).all() +s = j['sq_logit'].to_numpy().astype(np.float64) +z8 = (1 - WSQ) * z + WSQ * s; p = 1 / (1 + np.exp(-z8)); assert np.isfinite(p).all() and (p >= 0).all() and (p <= 1).all() +print('corr base/slow', round(float(np.corrcoef(c['base'], c['slow'])[0, 1]), 4), 'corr z/sq', round(float(np.corrcoef(z, s)[0, 1]), 4)) +pl.DataFrame({'rowid': m['rowid'], 'p': p}).write_parquet('eval_pred.parquet'); print('wrote eval_pred.parquet', m.height) diff --git a/v12/sol_work/src10/build_folds.py b/v12/sol_work/src10/build_folds.py new file mode 100644 index 0000000000000000000000000000000000000000..8b46fcac55147d81f68085ca2f720d384df3e830 --- /dev/null +++ b/v12/sol_work/src10/build_folds.py @@ -0,0 +1,31 @@ +import polars as pl, sys, os +sys.path.insert(0,'@@WORK@@') +from feats import load, first_events, build, log +D='@@WORK@@'; os.makedirs(f'{D}/feat',exist_ok=True) +TE,TR,EE,ER,NI=load(D) +DAY=86400; J20=1737331200 +END=J20+7*DAY; VC=J20+4*DAY +# validation: V = first event/user in [Jan24,Jan27); training folds end before Jan24 +specs=[(VC,END-VC,'V')]+[(J20+k*DAY,VC-(J20+k*DAY),f'g{k}') for k in (1,2,3)] +if len(sys.argv)>1 and sys.argv[1]=='c25': + specs=[(J20+5*DAY,2*DAY,'val25')]+[(J20+k*DAY,DAY,f'h{k}') for k in (1,2,3,4)] +if len(sys.argv)>1 and sys.argv[1]=='c26': + specs=[(J20+6*DAY,DAY,'val26')]+[(J20+k*DAY,DAY,f'h{k}') for k in (5,)]+[('eval',None,'eval')] +if len(sys.argv)>1 and sys.argv[1]=='eft': + specs=[(J20+k*DAY,(J20+5*DAY)-(J20+k*DAY),f'e{k}') for k in (1,2,3,4)]+[(J20+k*DAY,END-(J20+k*DAY),f'E{k}') for k in (1,2,3,4,5,6)] +if len(sys.argv)>1 and sys.argv[1]=='dft': + specs=[(J20+k*DAY,(J20+6*DAY)-(J20+k*DAY),f'd{k}') for k in (1,2,3,4,5)] +if len(sys.argv)>1 and sys.argv[1]=='final': + specs=[(J20+k*DAY,END-(J20+k*DAY),f'F{k}') for k in (1,2,3,4,5,6)]+[('eval',None,'eval')] +if len(sys.argv)>1 and sys.argv[1]=='lite': # v4_lite: only what the final fit needs (h1..h5 + val26 = f1..f6, eval) + specs=[(J20+k*DAY,DAY,f'h{k}') for k in (3,4,5)]+[(J20+6*DAY,DAY,'val26'),('eval',None,'eval')] +for C,W,name in specs: + out=f'{D}/feat/{name}.parquet' + if os.path.exists(out): continue + if C=='eval': + C=J20+7*DAY; T_ev=EE; T_rows=ER + else: + T_ev=first_events(TE,C,W); T_rows=TR.join(T_ev.select('eid'),on='eid') + R=build(C,T_ev,T_rows,TE,TR,NI) + R.drop('h','h_ts').write_parquet(out) + log('wrote',name,R.shape) diff --git a/v12/sol_work/src10/cl.py b/v12/sol_work/src10/cl.py new file mode 100644 index 0000000000000000000000000000000000000000..1fb0cbbff8cd89070494d2d69373b9ccb1257a19 --- /dev/null +++ b/v12/sol_work/src10/cl.py @@ -0,0 +1,41 @@ +"""Shared loader + feature-matrix cache (step C). Matrices cached as parquet keyed by hash(table names, feature list, sample).""" +import os, sys, hashlib, json, gc, numpy as np, polars as pl +sys.path.insert(0,'@@WORK@@'); sys.path.insert(1,'@@WORK@@/v3') +from feats import DROP +from features import feature_cols, GROUPS +D='@@WORK@@'; P2=f'{D}/feat'; P3=f'{D}/feat_v3'; CACHE=f'{D}/cache_mx'; os.makedirs(CACHE,exist_ok=True) +V2_OF={'f1':'h1','f2':'h2','f3':'h3','f4':'h4','f5':'h5','f6':'val26','val25':'val25','val26':'val26','eval':'eval'} +V2_OF.update({f'{p}{k}':f'{p}{k}' for p in 'eEd' for k in range(1,7)}) +TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour'] +KEEP=['rowid','eid','u','ts','clicked','us_cold','us_nohist','tm_stale'] +def load_tab(n, extra=(), variant='notime'): + a=pl.read_parquet(f'{P3}/{n}.parquet'); b=pl.read_parquet(f'{P2}/{V2_OF[n]}.parquet') + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.read_parquet(f'{D}/{sub}/{n}.parquet'),on='rowid',how='left') + if '+' in variant: + raise ValueError('mods not used in v7') + return a +def featlist(df, variant='notime'): + f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith(('v2_','x_','r_','s_','ev2_','rc_','sh_','hs_','lr','tA_','tB_'))] + if 'notime' in variant: f=[c for c in f if c not in TIME] + if variant.split('+')[0]=='stale': f=[c for c in f if c not in TIME or c=='tm_stale'] + return f +def usample(df, frac): + if frac>=1: return df + return df.filter((pl.col('u').hash(seed=7) % 10000) < int(frac*10000)) +def matrix(n, frac=1.0, variant='notime', extra=()): + """returns (X float32, meta DataFrame, feats); cached.""" + key=hashlib.md5(json.dumps([n,frac,variant,list(extra),[os.path.getmtime(f) for f in (f'{P3}/{n}.parquet',f'{P2}/{V2_OF[n]}.parquet',f'{D}/feat_x/{n}.parquet')+((f'{D}/feat_x2/{n}.parquet',) if os.path.isdir(f'{D}/feat_x2') else ())]+[os.path.getmtime(f'{D}/{x}/{n}.parquet') for x in extra]]+([os.environ['XDIR']] if os.environ.get('XDIR','feat_x')!='feat_x' else [])).encode()).hexdigest()[:12] + fp=f'{CACHE}/{n}_{key}.parquet' + if os.path.exists(fp): + t=pl.read_parquet(fp); feats=[c for c in t.columns if c not in KEEP or c in ('us_cold','us_nohist')] + fl=json.load(open(fp+'.json')); + else: + t=usample(load_tab(n,extra,variant),frac); fl=featlist(t,variant) + t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl))) + t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP]) + t.write_parquet(fp); json.dump(fl,open(fp+'.json','w')) + X=t.select(fl).to_numpy().astype(np.float32); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect() + return X, meta, fl diff --git a/v12/sol_work/src10/ens.py b/v12/sol_work/src10/ens.py new file mode 100644 index 0000000000000000000000000000000000000000..b93269ced2923450ebecffe5fef00f722d1559fe --- /dev/null +++ b/v12/sol_work/src10/ens.py @@ -0,0 +1,36 @@ +"""Phase 1 ensemble components. python ens.py CUT TAG [--seeds 0,1] [--folds f|fe] [--stale] [--rc] [--drop re] [--params json] [--rounds 400] +CUT=25/26: train f1..f(CUT-21) (+e/d folds) full data, predict val{CUT}; CUT=27: train f1..f6 (+E1..E6), predict eval. +Saves ens/{tag}_c{CUT}_s{seed}.npy with raw scores at rounds 300,400(,500) (columns).""" +import sys, re, json, gc, time, argparse, numpy as np, lightgbm as lgb +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--seeds',default='0') +ap.add_argument('--folds',default='f'); ap.add_argument('--stale',action='store_true'); ap.add_argument('--rc',action='store_true') +ap.add_argument('--drop',default=None); ap.add_argument('--params',default='{}'); ap.add_argument('--rounds',type=int,default=400); ap.add_argument('--threads',type=int,default=5); ap.add_argument('--lr',default=''); ap.add_argument('--nfolds',type=int,default=99); ap.add_argument('--negfrac',type=float,default=1.0); ap.add_argument('--keep',default='') +a=ap.parse_args(); t0=time.time() +extra=(('feat_rc',) if a.rc else ())+tuple(x for x in a.lr.split(',') if x); var='stale' if a.stale else 'notime' +efold={25:[f'e{k}' for k in range(1,5)],26:[f'd{k}' for k in range(1,6)],27:[f'E{k}' for k in range(1,7)]}[a.cut] +trn=([f'f{k}' for k in range(max(1,a.cut-20-a.nfolds),a.cut-20)] if 'f' in a.folds else [])+(efold if 'e' in a.folds else []) +Xs=[];ys=[];feats=None +for n in trn: + X,m,f=matrix(n,1.0,variant=var,extra=extra); feats=f; Xs.append(X); ys.append(m['clicked'].to_numpy()); del X,m; gc.collect() +X=np.concatenate(Xs); del Xs; gc.collect(); y=np.concatenate(ys) +keep=[i for i,c in enumerate(feats) if not (a.drop and re.search(a.drop,c))]; fl=[feats[i] for i in keep] +if a.keep: KS=set(open(a.keep).read().split()); keep=[i for i in keep if feats[i] in KS]; fl=[feats[i] for i in keep] +if a.negfrac<1: + rs=np.random.RandomState(123); sel=(y==1)|(rs.rand(len(y)) v2work/feat_x/NAME.parquet + - con variants: k=32/128, recency-weighted (tau=2d) click/non-click centroids, item-side con (users who clicked i vs skipped i) + - user-cluster item CTR (kmeans on user SVD vectors, 64 & 256 clusters), smoothed toward item CTR + - within-event z-score and rank for each""" +import sys, os, time, numpy as np, polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +from sklearn.cluster import MiniBatchKMeans +sys.path.insert(0, '@@WORK@@/v3') +from features import load, click_log +from split import J20, DAY, first_events +OUT = os.environ.get('X_OUT', '@@WORK@@/feat_x'); os.makedirs(OUT, exist_ok=True) +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'), + **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f'd{k}': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}} +def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9) +def csr(u, i, w): return sparse.csr_matrix((w.astype(np.float32), (u, i)), shape=(NU, NI)) +def build(C, T_ev, T_rows): + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CL = click_log(C, P_ev, P_rows, T_ev) # user-level: train data + each target row's own history + CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # global/item-level: train data only + assert CL['ts'].max() < C and P_rows['ts'].max() < C + R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii') + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + cg = CLg.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = csr(cg['u'].to_numpy(), cg['ii'].to_numpy(), np.ones(cg.height)); Scg = Xc.copy() + cnt = CL.group_by('u', 'ii').agg(pl.len().alias('n'), pl.col('ts').max().alias('lt')) + cu = cnt['u'].to_numpy(); ci = cnt['ii'].to_numpy() + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc + Xu = csr(cu, ci, np.ones(cnt.height)) @ sparse.diags(idf); Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + neg = P_rows.filter(pl.col('clicked') == 0) + nu_ = neg['u'].to_numpy(); ni_ = neg['ii'].to_numpy() + Sc = csr(cu, ci, np.ones(cnt.height)); Sn = csr(nu_, ni_, np.ones(neg.height)) + rc = np.exp(-(C - cnt['lt'].to_numpy()) / (2 * DAY)); rn = np.exp(-(C - neg['ts'].to_numpy()) / (2 * DAY)) + Scr = csr(cu, ci, rc); Snr = csr(nu_, ni_, rn) + F = {} + for k in (32, 64, 128): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + Vn = nrm(Vt.T * np.sqrt(S)); Un = nrm(np.asarray(Xu @ (Vt.T / np.sqrt(S)))); cand = Vn[ii] # user fold-in + nC = nrm(np.asarray(Sc @ Vn)); nN = nrm(np.asarray(Sn @ Vn)) + F[f'x_con05_k{k}'] = (nrm(nC - 0.5 * nN)[uu] * cand).sum(1) + if k == 64: + nCr = nrm(np.asarray(Scr @ Vn)); nNr = nrm(np.asarray(Snr @ Vn)) + for a, nm in ((0.5, '05'), (1.0, '1')): + F[f'x_conr{nm}_k64'] = (nrm(nCr - a * nNr)[uu] * cand).sum(1) + F['x_crcos_k64'] = (nCr[uu] * cand).sum(1) + # item side: centroid of users who clicked item minus users who skipped it, vs target user vector + # item side / clusters: user vectors from TRAIN-ONLY clicks (Ug), so no target (eval) histories enter item-level stats + Ug = nrm(np.asarray(Xc @ (Vt.T / np.sqrt(S)))) + iC = nrm(np.asarray(Scg.T @ Ug)); iN = nrm(np.asarray(Sn.T @ Ug)) + F['x_icon05_k64'] = (nrm(iC - 0.5 * iN)[ii] * Un[uu]).sum(1) + F['x_icos_k64'] = (iC[ii] * Un[uu]).sum(1) + # user-cluster CTR + act = np.unique(np.concatenate([P_rows['u'].to_numpy(), cg['u'].to_numpy()])) + for nc in (64, 256): + km = MiniBatchKMeans(nc, random_state=0, batch_size=4096, n_init=3).fit(Ug[act]) + lab = km.predict(Ug).astype(np.int32) # train users' clusters (train-only vectors) -> cluster CTR tables + labt = km.predict(Un[uu]).astype(np.int32) # target row's cluster from its own user vector (own history allowed) + g = float(P_rows['clicked'].mean()) + pr = P_rows.select('ii', 'clicked', pl.Series('cl', lab[P_rows['u'].to_numpy()])) + ict = pr.group_by('ii').agg(((pl.col('clicked').sum() + 20 * g) / (pl.len() + 20)).alias('ictr')) + cct = pr.group_by('cl', 'ii').agg(pl.col('clicked').sum().alias('c'), pl.len().alias('n')) + q = pl.DataFrame({'cl': labt, 'ii': ii.astype(np.int32), 'r': np.arange(len(uu))}).join(cct, on=['cl', 'ii'], how='left') \ + .join(ict, on='ii', how='left').sort('r').with_columns(pl.col('ictr').fill_null(g), pl.col('c').fill_null(0), pl.col('n').fill_null(0)) + F[f'x_clctr{nc}'] = ((q['c'] + 10 * q['ictr']) / (q['n'] + 10)).to_numpy() + F[f'x_clctr{nc}_lift'] = F[f'x_clctr{nc}'] / q['ictr'].to_numpy() + F[f'x_climp{nc}'] = q['n'].to_numpy().astype(np.float32) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()]) + zc = [c for c in F if not c.startswith('x_climp')] + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in zc] + + [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in zc]) + return R.drop('eid', 'u', 'ii') +for name in sys.argv[1:]: + t0 = time.time(); C, W = SPEC[name] + if W == 'eval': R = build(C, EE, ER) + else: + T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/feats.py b/v12/sol_work/src10/feats.py new file mode 100644 index 0000000000000000000000000000000000000000..652a49954e219d12f1077ea5f88c70571091285e --- /dev/null +++ b/v12/sol_work/src10/feats.py @@ -0,0 +1,224 @@ +"""Feature builder for TechArena CTR. All stats for a cutoff C use only events with ts < C +(plus the target events' own histories, which by construction contain only clicks < their ts).""" +import polars as pl, numpy as np, time +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +DAY = 86400 +K_SMOOTH = 20 + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/train_events.parquet') + TR = pl.read_parquet(f'{D}/train_rows.parquet') + EE = pl.read_parquet(f'{D}/evaluation_events.parquet') + ER = pl.read_parquet(f'{D}/evaluation_rows.parquet') + # global item index + items = pl.concat([TR['item'], ER['item'], TE['h_items'].explode().drop_nulls(), EE['h_items'].explode().drop_nulls()]).unique().sort() + imap = pl.DataFrame({'item': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)}) + def mapi(df, col='item'): + return df.join(imap.rename({'item': col, 'ii': col + '_ii'}), on=col, how='left') + TR = mapi(TR).drop('item').rename({'item_ii': 'ii'}) + ER = mapi(ER).drop('item').rename({'item_ii': 'ii'}) + def map_hist(E): + ex = E.select('eid', 'h_items').explode('h_items').with_row_index('r') + ex = ex.join(imap.rename({'item': 'h_items'}), on='h_items', how='left').sort('r') + h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').drop_nulls().alias('h')) + return E.drop('h_items').join(h, on='eid', how='left').with_columns(pl.col('h').fill_null([])) + TE = map_hist(TE); EE = map_hist(EE) + return TE, TR, EE, ER, len(items) + +def first_events(TE, C, W): + """first event per user in [C, C+W)""" + return (TE.filter((pl.col('ts') >= C) & (pl.col('ts') < C + W)).sort('ts') + .group_by('user', maintain_order=True).first()) + +def build(C, T_ev, T_rows, TE, TR, NI, svd_k=64, with_label=True): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'user', 'ts', 'tb'), on='eid', how='inner') + g = P_rows['clicked'].mean() if len(P_rows) else 0.063 + ndays = max((C - P_ev['ts'].min()) / DAY, 1e-3) if len(P_ev) else 1e-3 + + # ---------- click pool (user, item, ts) ---------- + def hist_clicks(E): + return (E.select('user', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls() + .rename({'h': 'ii', 'h_ts': 'ts'})) + pool = pl.concat([hist_clicks(P_ev), hist_clicks(T_ev), + P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique() + pool = pool.with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64)) + # global/item-level pool: train data only (no target-event histories, so nothing aggregates across eval rows); + # `pool` (with each target row's own history) is used only for that user's own features + poolg = pl.concat([hist_clicks(P_ev), P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique() \ + .with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64)) + log('pool', len(pool)) + + # ---------- item stats ---------- + it = P_rows.group_by('ii').agg(pl.len().alias('i_imp'), pl.col('clicked').sum().alias('i_clk'), + (pl.col('ts') >= C - DAY).sum().alias('i_imp1d'), + pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().alias('i_clk1d')) + it = it.with_columns(((pl.col('i_clk') + K_SMOOTH * g) / (pl.col('i_imp') + K_SMOOTH)).alias('i_ctr'), + ((pl.col('i_clk1d') + K_SMOOTH * g) / (pl.col('i_imp1d') + K_SMOOTH)).alias('i_ctr1d'), + (pl.col('i_imp') / ndays).alias('i_imp_rate')) + ip = poolg.group_by('ii').agg(pl.len().alias('ip_n'), pl.col('user').n_unique().alias('ip_users'), + (pl.col('ts') >= C - DAY).sum().alias('ip_1d'), + (pl.col('ts') >= C - 3 * DAY).sum().alias('ip_3d'), + ((pl.col('ts') - C) / DAY).exp().sum().alias('ip_dec1'), + ((pl.col('ts') - C) / (3 * DAY)).exp().sum().alias('ip_dec3'), + (pl.col('ts') < 1737331200).sum().alias('ip_warm')) + itb = P_rows.group_by('ii', 'tb').agg(pl.len().alias('n'), pl.col('clicked').sum().alias('c')) + itb = itb.with_columns(((pl.col('c') + K_SMOOTH * g) / (pl.col('n') + K_SMOOTH)).alias('i_ctr_tb')).select('ii', 'tb', 'i_ctr_tb') + + # ---------- user stats ---------- + ue = P_rows.group_by('user').agg(pl.col('eid').n_unique().alias('u_ev'), pl.col('clicked').sum().alias('u_clk'), + pl.col('ts').max().alias('u_last_ev')) + ue = ue.with_columns(((pl.col('u_clk') + 5 * g * 10) / (pl.col('u_ev') + 5)).alias('u_ctr')) + up = pool.group_by('user').agg(pl.len().alias('up_n'), pl.col('ts').max().alias('up_last'), + pl.col('ii').n_unique().alias('up_uniq'), + (pl.col('ts') >= C - DAY).sum().alias('up_1d')) + # user-item prior impressions + ui = P_rows.group_by('user', 'ii').agg(pl.len().alias('ui_imp'), pl.col('clicked').sum().alias('ui_clk'), + pl.col('ts').max().alias('ui_last_imp')) + uip = pool.group_by('user', 'ii').agg(pl.len().alias('uip_n'), pl.col('ts').max().alias('uip_last')) + + # ---------- co-occurrence (user sets) & transitions ---------- + us = poolg.select('user', 'ii').unique() + # v8: user-set co-occurrence counts via sparse B^T B (same integer counts as the former self-join, far less memory) + _ur = us['user'].rank('dense').to_numpy().astype(np.int64) - 1 + _B = sparse.csr_matrix((np.ones(len(us), np.int32), (_ur, us['ii'].to_numpy().astype(np.int64))), shape=(int(_ur.max()) + 1 if len(us) else 0, NI)) + _C = (_B.T @ _B).tocsr(); _pop = _C.diagonal().astype(np.float64); _C.setdiag(0); _C.eliminate_zeros() + del _B, _ur + log('cooc pairs', _C.nnz) # _C / _pop replace the (ii, ii_b, co, cos) frame + hash join + seq = poolg.sort('user', 'ts') + tr_pairs = [] + for lag in (1, 2, 3): + tr_pairs.append(seq.with_columns(pl.col('ii').shift(lag).over('user').alias('prev')) + .drop_nulls('prev').select(pl.col('prev').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(tr_pairs).group_by('prev', 'ii').agg(pl.col('w').sum().alias('trn')) + trn_out = trn.group_by('prev').agg(pl.col('trn').sum().alias('trn_tot')) + trn = trn.join(trn_out, on='prev').with_columns((pl.col('trn') / (pl.col('trn_tot') + 5)).cast(pl.Float32).alias('trp')).select('prev', 'ii', 'trn', 'trp') + + # ---------- SVD embeddings of user x item click matrix ---------- + uidx = us.select('user').unique().with_row_index('ur') + m = us.join(uidx, on='user') + df_ = np.bincount(m['ii'].to_numpy(), minlength=NI) + idf = np.log((len(uidx) + 1) / (df_ + 1)).astype(np.float32) + X = sparse.csr_matrix((idf[m['ii'].to_numpy()], (m['ur'].to_numpy(), m['ii'].to_numpy())), shape=(len(uidx), NI)) + rn = np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel()) + 1e-6 + X = sparse.diags(1 / rn) @ X + U, S, Vt = randomized_svd(X, svd_k, random_state=0, n_iter=4) + V = Vt.T.astype(np.float32) # unscaled singular vectors (best in emb_exp) + V /= (np.linalg.norm(V, axis=1, keepdims=True) + 1e-6) + V32 = V[:, :32] / (np.linalg.norm(V[:, :32], axis=1, keepdims=True) + 1e-6) + # fold-in of every target user's own pool (train clicks + own history) onto the train-only factors + uidx = pool.select('user').unique().with_row_index('ur') + mu = pool.select('user', 'ii').unique().join(uidx, on='user') + Xu = sparse.csr_matrix((idf[mu['ii'].to_numpy()], (mu['ur'].to_numpy(), mu['ii'].to_numpy())), shape=(len(uidx), NI)) + Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + U = np.asarray(Xu @ (Vt.T / S)).astype(np.float32) + S = np.ones_like(S) + log('svd done') + + # ---------- assemble target rows ---------- + R = T_rows.join(T_ev, on='eid', how='left').with_row_index('r') + R = R.with_columns((pl.col('ts') - C).alias('stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('hour')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(itb, on=['ii', 'tb'], how='left') + .join(ue, on='user', how='left').join(up, on='user', how='left') + .join(ui, on=['user', 'ii'], how='left').join(uip, on=['user', 'ii'], how='left').sort('r')) + R = R.with_columns( + pl.col('h').list.len().alias('h_len'), + (pl.col('ts') - pl.col('h_ts').list.last()).alias('h_since_last'), + (pl.col('ts') - pl.col('h_ts').list.first()).alias('h_since_first'), + pl.col('h').list.n_unique().alias('h_uniq'), + pl.col('h_ts').list.eval((pl.element() >= pl.element().max() - DAY).sum()).list.first().alias('h_1d_before_last'), + (pl.col('ts') - pl.col('u_last_ev')).alias('u_since_ev'), + (pl.col('ts') - pl.col('up_last')).alias('up_since'), + (pl.col('ts') - pl.col('ui_last_imp')).alias('ui_since_imp'), + (pl.col('ts') - pl.col('uip_last')).alias('uip_since'), + (pl.col('ui_imp') - pl.col('ui_clk')).alias('ui_noclk'), + (pl.col('ip_n') / (pl.col('i_imp') + 1)).alias('ip_per_imp'), + (pl.col('i_ctr_tb') - pl.col('i_ctr')).alias('i_ctr_tb_diff'), + ) + R = R.with_columns((pl.col('h_since_first') / (pl.col('h_len') - 1).clip(1)).alias('h_gap_mean'), + (pl.col('h_uniq') / pl.col('h_len').clip(1)).alias('h_uniq_ratio')) + + # pairwise candidate x history + PR = (R.select('r', 'ii', 'h', 'h_ts', 'ts').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'h_ts', 'pos']).drop_nulls('h') + .with_columns(pl.col('h').cast(pl.Int32))) + _pi = PR['ii'].to_numpy().astype(np.int64); _ph = PR['h'].to_numpy().astype(np.int64) + _cv = np.asarray(_C[_pi, _ph]).ravel().astype(np.float64) + _cs = np.where(_cv > 0, _cv / np.sqrt(np.maximum(_pop[_pi] * _pop[_ph], 1e-12)), 0.0).astype(np.float32) + PR = PR.with_columns(pl.Series('co', _cv.astype(np.int32)), pl.Series('cos', _cs)); del _pi, _ph, _cv, _cs + PR = PR.with_columns((0.85 ** (pl.col('pos') - 1)).alias('rw'), (pl.col('h') == pl.col('ii')).alias('same')) + agg = PR.group_by('r').agg( + pl.col('cos').sum().alias('co_sum'), pl.col('cos').max().alias('co_max'), pl.col('cos').mean().alias('co_mean'), + (pl.col('cos') * pl.col('rw')).sum().alias('co_rw'), pl.col('co').sum().alias('co_cnt'), + pl.col('cos').filter(pl.col('pos') <= 3).sum().alias('co_last3'), + pl.col('same').sum().alias('inh_cnt'), pl.col('pos').filter(pl.col('same')).min().alias('inh_pos'), + (pl.col('ts') - pl.col('h_ts')).filter(pl.col('same')).min().alias('inh_since'), + (pl.col('cos') > 0).sum().alias('co_nz')) + R = R.join(agg, on='r', how='left') + # transitions from last 3 history items + TRp = (R.select('r', 'ii', pl.col('h').list.tail(3).alias('h3')).with_columns(pl.int_ranges(pl.col('h3').list.len(), 0, -1).alias('pos')) + .explode(['h3', 'pos']).drop_nulls('h3').rename({'h3': 'prev'}).with_columns(pl.col('prev').cast(pl.Int32)) + .join(trn, on=['prev', 'ii'], how='left').with_columns(pl.col('trn').fill_null(0), pl.col('trp').fill_null(0))) + tagg = TRp.group_by('r').agg(pl.col('trn').filter(pl.col('pos') == 1).sum().alias('tr_last'), + pl.col('trp').filter(pl.col('pos') == 1).sum().alias('trp_last'), + pl.col('trn').sum().alias('tr_last3'), pl.col('trp').sum().alias('trp_last3')) + R = R.join(tagg, on='r', how='left').sort('r') + # SVD cosine: recency weighted mean of history vectors + ex = PR.select('r', 'h', 'rw') + Wrh = sparse.csr_matrix((ex['rw'].to_numpy().astype(np.float32), (ex['r'].to_numpy(), ex['h'].to_numpy())), shape=(len(R), NI)) # v8: replaces np.add.at + hv = np.asarray(Wrh @ V, dtype=np.float32) + hvn = hv / (np.linalg.norm(hv, axis=1, keepdims=True) + 1e-6) + cand = V[R['ii'].to_numpy()] + svd_cos = (hvn * cand).sum(1) + # user factor from SVD (pool-based, covers clicks beyond last 20) + uf = np.zeros((len(R), V.shape[1]), np.float32) + um = R.select('user').join(uidx, on='user', how='left')['ur'].to_numpy() + ok = ~np.isnan(um.astype(float)) if um.dtype.kind == 'f' else np.ones(len(um), bool) + umi = np.nan_to_num(um.astype(float), nan=0).astype(np.int64) + ufv = (U[umi] * np.sqrt(S)).astype(np.float32); ufv[~ok] = 0 + ufv /= (np.linalg.norm(ufv, axis=1, keepdims=True) + 1e-6) + svd_ucos = (ufv * cand).sum(1) + hv32 = np.asarray(Wrh @ V32, dtype=np.float32) + svd32_cos = (hv32 / (np.linalg.norm(hv32, axis=1, keepdims=True) + 1e-6) * V32[R['ii'].to_numpy()]).sum(1) + extra_svd = {} + for kk in (16, 24, 48): + Vk = V[:, :kk] / (np.linalg.norm(V[:, :kk], axis=1, keepdims=True) + 1e-6) + hk = np.asarray(Wrh @ Vk, dtype=np.float32) + extra_svd[f'svd{kk}_cos'] = (hk / (np.linalg.norm(hk, axis=1, keepdims=True) + 1e-6) * Vk[R['ii'].to_numpy()]).sum(1) + # pairwise svd similarity: max over history, and with the most recent item + pr_r = PR['r'].to_numpy(); pr_c = PR['ii'].to_numpy(); pr_h = PR['h'].to_numpy() + sims = np.empty(len(PR), np.float32) + for k in range(0, len(PR), 2_000_000): + sl = slice(k, k + 2_000_000); sims[sl] = (V[pr_c[sl]] * V[pr_h[sl]]).sum(1) + sp = pl.DataFrame({'r': pr_r, 'sim': sims, 'pos': PR['pos'].to_numpy(), 'same': PR['same'].to_numpy()}).filter(~pl.col('same')) + sagg = sp.group_by('r').agg(pl.col('sim').max().alias('svd_maxsim'), pl.col('sim').filter(pl.col('pos') == 1).max().alias('svd_last'), + pl.col('sim').top_k(3).mean().alias('svd_top3')) + R = R.with_columns(pl.Series('svd_cos', svd_cos), pl.Series('svd_ucos', svd_ucos), pl.Series('svd_hnorm', np.linalg.norm(hv, axis=1)), + pl.Series('svd32_cos', svd32_cos), *[pl.Series(k, v) for k, v in extra_svd.items()]) + R = R.join(sagg, on='r', how='left').sort('r') + + fill0 = ['i_imp', 'i_clk', 'i_imp1d', 'i_clk1d', 'ip_n', 'ip_users', 'ip_1d', 'ip_3d', 'ip_dec1', 'ip_dec3', 'ip_warm', + 'u_ev', 'u_clk', 'up_n', 'up_uniq', 'up_1d', 'ui_imp', 'ui_clk', 'ui_noclk', 'uip_n', 'co_sum', 'co_max', 'co_mean', 'co_rw', + 'co_cnt', 'co_last3', 'inh_cnt', 'co_nz', 'tr_last', 'trp_last', 'tr_last3', 'trp_last3'] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns(pl.col('i_ctr').fill_null(g), pl.col('i_ctr1d').fill_null(g), pl.col('i_ctr_tb').fill_null(g), pl.col('u_ctr').fill_null(g * 10)) + # within-event relative features + rel = ['i_ctr', 'ip_dec1', 'ip_n', 'co_sum', 'co_rw', 'co_max', 'svd_cos', 'svd_ucos', 'svd32_cos', 'svd16_cos', 'svd24_cos', 'svd48_cos', 'svd_maxsim', 'svd_top3', 'tr_last3', 'uip_n', 'i_ctr_tb', 'ui_noclk'] + R = R.with_columns([pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in rel] + + [(pl.col(c) - pl.col(c).mean().over('eid')).alias(c + '_dm') for c in rel] + + [(pl.col(c) / (pl.col(c).max().over('eid') + 1e-9)).alias(c + '_rmax') for c in ['co_sum', 'svd_cos', 'ip_dec1', 'co_rw']]) + R = R.with_columns(pl.col('co_rw').max().over('eid').alias('ev_co_max'), pl.col('svd_cos').max().over('eid').alias('ev_svd_max'), + pl.col('svd_cos').mean().over('eid').alias('ev_svd_mean'), + (pl.col('inh_cnt') > 0).sum().over('eid').alias('ev_n_inh'), pl.col('i_ctr').mean().over('eid').alias('ev_ictr_mean'), + pl.col('ip_dec1').mean().over('eid').alias('ev_ipdec_mean')) + log(f'build C={C} rows={len(R)} {time.time()-t0:.0f}s') + starter = dict(g=g) + return R + +DROP = ['r', 'rowid', 'eid', 'user', 'ii', 'h', 'h_ts', 'ts', 'clicked', 'u_last_ev', 'up_last', 'ui_last_imp', 'uip_last', 'fold'] diff --git a/v12/sol_work/src10/lrfeat2.py b/v12/sol_work/src10/lrfeat2.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee3d585ef8fbe5cf89701ac362aba3692379ce7 --- /dev/null +++ b/v12/sol_work/src10/lrfeat2.py @@ -0,0 +1,60 @@ +"""F1 (data_dig_v1): history-centroid cosine in an exact low-rank SVD space. Train-only, rebuilt at each fold cutoff C. +Matrix: users x items, binary, rows = each user's last N distinct history clicks (from train events with ts < C), +columns scaled by 1/sqrt(pop+5); exact svds (ARPACK) k=17; item vectors = comps 2..K+1 (popularity comp dropped) * s. +Feature: cos(candidate vector, sum of own-history normalised vectors). Within-event _z and _dmax. Eval: fitted at J27 on train only. +Usage: python lrfeat.py NAME... -> feat_lr/NAME.parquet (rowid, lr{K}_cos/_z/_dmax for K in 10,12,16)""" +import sys, os, time, numpy as np, polars as pl, scipy.sparse as sp +from scipy.sparse.linalg import svds +sys.path.insert(0, '@@WORK@@/v3') +from features import load +from split import J20, DAY, first_events +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +N = int(os.environ.get('LR_N', 40)); OUT = os.environ.get('LR_OUT', '@@WORK@@/feat_lr2'); os.makedirs(OUT, exist_ok=True) +KS = tuple(int(x) for x in os.environ.get('LR_KS', '10,12,16').split(',')) +CON = int(os.environ.get('LR_CON', 0)) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None)} +_V = {} +def vecs(C): + if C in _V: return _V[C] + h = (TE.filter(pl.col('ts') < C).select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('t')).unique().sort('u', 't', 'ii')) + h = h.with_columns(pl.int_range(pl.len()).over('u').alias('p'), pl.len().over('u').alias('L')).filter(pl.col('L') - pl.col('p') <= N).unique(['u', 'ii']) + M = sp.csr_matrix((np.ones(h.height, np.float32), (h['u'].to_numpy(), h['ii'].to_numpy())), shape=(NU, NI)) + pop = np.asarray(M.sum(0)).ravel(); M = M.multiply(1 / np.sqrt(pop + 5)).tocsr() + _, s, vt = svds(M, k=max(max(KS) + 1, 17), random_state=0); o = np.argsort(-s); s, vt = s[o], vt[o] + print(time.strftime('%T'), 'C', C, 'pairs', h.height, 'sv', np.round(s[:15], 2), flush=True) + _V[C] = (s, vt); return _V[C] +def build(C, T_ev, T_rows): + s, vt = vecs(C) + E = T_ev.select('eid', 'h').with_row_index('k'); kk = E.select('eid', 'k') + H = E.select('k', 'h').explode('h').drop_nulls('h'); hk = H['k'].to_numpy(); hi = H['h'].to_numpy().astype(np.int64) + R = T_rows.select('rowid', 'eid', 'ii').join(kk, on='eid', how='left'); rk = R['k'].to_numpy(); ri = R['ii'].to_numpy().astype(np.int64) + F = {} + if CON: + P_ev = TE.filter(pl.col('ts') < C).select('eid', 'u') + ng = TR.filter(pl.col('clicked') == 0).join(P_ev, on='eid', how='inner').select('u', 'ii').unique() + ue = T_ev.select('u').with_row_index('k').join(ng, on='u', how='inner'); nk = ue['k'].to_numpy(); ni = ue['ii'].to_numpy().astype(np.int64) + Shk = sp.csr_matrix((np.ones(len(hk)), (hk, hi)), shape=(E.height, NI)) + if CON: Snk = sp.csr_matrix((np.ones(len(nk)), (nk, ni)), shape=(E.height, NI)) + for K in KS: + V = (vt[1:K + 1].T * s[1:K + 1]); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9) + hc = np.asarray(Shk @ V) # v8: sparse matmul replaces np.add.at + hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1) + x = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan) + F[f'lr{K}_cos'] = x.astype(np.float32) + if CON: + nc = np.asarray(Snk @ V) + hn = hc / (np.linalg.norm(hc, axis=1, keepdims=True) + 1e-9); nn = nc / (np.linalg.norm(nc, axis=1, keepdims=True) + 1e-9) + for al, nm in ((0.5, '05'), (1.0, '1')): + d = (hn - al * nn)[rk]; dn = np.linalg.norm(d, axis=1) + F[f'lr{K}_con{nm}'] = np.where(dn > 0, (V[ri] * d).sum(1) / (dn + 1e-9), np.nan).astype(np.float32) + R = R.select('rowid', 'eid').with_columns([pl.Series(c, v).fill_nan(None) for c, v in F.items()]) + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c.replace('_cos', '_z') if c.endswith('_cos') else c + '_z') for c in F] + + [(pl.col(c) - pl.col(c).max().over('eid')).alias(c.replace('_cos', '_dmax') if c.endswith('_cos') else c + '_dmax') for c in F]) + return R.drop('eid') +for n in sys.argv[1:]: + t0 = time.time() + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{n}.parquet'); print(n, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/ppmi.py b/v12/sol_work/src10/ppmi.py new file mode 100644 index 0000000000000000000000000000000000000000..0d2fbf205f870114ac1dcb6006f1bc36ad22a94a --- /dev/null +++ b/v12/sol_work/src10/ppmi.py @@ -0,0 +1,59 @@ +"""v8 (ported from v5 ppmi.py): sequence-window PPMI-SVD item embeddings from CLg only (train clicks + train-event histories, ts WORK/feat_pp/NAME.parquet""" +import sys, os, time, json, numpy as np, polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +sys.path.insert(0, '@@WORK@@/v3') +from features import load, click_log +from split import J20, DAY, first_events +OUT = '@@WORK@@/feat_pp'; os.makedirs(OUT, exist_ok=True) +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val23': (J20 + 3 * DAY, None), 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval')} +def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9) +def csr(u, i, w, shape): return sparse.csr_matrix((np.asarray(w, np.float32), (u, i)), shape=shape) +def ppmi_emb(CLg, ks=(32, 64)): + s = CLg.select('u', pl.col('ii').cast(pl.Int64)).to_numpy() # sorted by u, ts + u, it = s[:, 0], s[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32) + for d in range(1, 6): + m = u[d:] == u[:-d]; a, b = it[:-d][m], it[d:][m]; ok = a != b; a, b = a[ok], b[ok] + A = A + csr(a, b, np.full(len(a), 1.0 / d), (NI, NI)) + csr(b, a, np.full(len(a), 1.0 / d), (NI, NI)) + A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum() + pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0 + M = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI)) + E = {} + for k in ks: + U, S, Vt = randomized_svd(M, k, random_state=0, n_iter=5); E[k] = nrm(U * np.sqrt(S)) + return E, int(len(a)) +def build(C, T_ev, T_rows): + P_ev = TE.filter(pl.col('ts') < C); P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CL = click_log(C, P_ev, P_rows, T_ev); CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) + assert CLg['ts'].max() < C and CL['ts'].max() < C + # acceptance: window counts from CLg only -> target events' histories must not change CLg + assert CLg.height == click_log(C, P_ev, P_rows, T_ev.head(0)).height + E, _ = ppmi_emb(CLg) + R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii') + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + cnt = CL.group_by('u', 'ii').agg(pl.len()); Sc = csr(cnt['u'].to_numpy(), cnt['ii'].to_numpy(), np.ones(cnt.height), (NU, NI)) + neg = P_rows.filter(pl.col('clicked') == 0); Sn = csr(neg['u'].to_numpy(), neg['ii'].to_numpy(), np.ones(neg.height), (NU, NI)) + F = {} + E32, E64 = E[32], E[64] + nC = nrm(np.asarray(Sc @ E32)); nN = nrm(np.asarray(Sn @ E32)); F['x_pp_con05_k32'] = (nrm(nC - 0.5 * nN)[uu] * E32[ii]).sum(1) + hC = nrm(np.asarray(Sc @ E64)); F['x_pp_hcos'] = (hC[uu] * E64[ii]).sum(1) + l3 = CL.sort('u', 'ts').group_by('u', maintain_order=True).tail(3).select('u', pl.col('ii').cast(pl.Int64)) + L = np.full((NU, 3), -1, np.int64); pos = l3.with_columns(pl.int_range(pl.len()).over('u').alias('p')) + L[pos['u'].to_numpy(), pos['p'].to_numpy()] = pos['ii'].to_numpy() + best = np.full(len(uu), -np.inf, np.float32); Lu = L[uu] + for j in range(3): + v = Lu[:, j]; ok = v >= 0; sc = np.full(len(uu), -np.inf, np.float32); sc[ok] = (E64[v[ok]] * E64[ii[ok]]).sum(1) + best = np.maximum(best, sc) + F['x_pp_last3'] = np.where(np.isfinite(best), best, 0.0) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()]) + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in F] + + [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in F]) + return R +def table(name): + C, W = SPEC[name] + if W == 'eval': return build(C, EE, ER) + T_ev = first_events(TE, C, W); return build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) +for name in sys.argv[1:]: + t0 = time.time(); R = table(name).drop('eid', 'u', 'ii'); R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/prep.py b/v12/sol_work/src10/prep.py new file mode 100644 index 0000000000000000000000000000000000000000..5de04f95f2309e945d7dfc0e9fe2d8598625e0eb --- /dev/null +++ b/v12/sol_work/src10/prep.py @@ -0,0 +1,22 @@ +import polars as pl, time, sys +D='@@WORK@@' +def hexid(e, n=15): # opaque id -> int64 via first 15 hex chars after prefix + return e.str.slice(4, n).str.to_integer(base=16) +def prep(name, has_label): + t=time.time() + lf=pl.scan_csv(f'{D}/{name}.csv', schema_overrides={'history_item_ids':pl.Utf8,'history_timestamps':pl.Utf8}) + cols=[pl.col('rowid'), hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('item_id')).alias('item')] + if has_label: cols.append(pl.col('clicked').cast(pl.Int8)) + lf.select(cols).sink_parquet(f'{D}/{name}_rows.parquet') + print(name,'rows done',time.time()-t,flush=True) + ev=(lf.select('event_id','user_id','timestamp','time_bucket','day_type','history_item_ids','history_timestamps') + .unique(subset='event_id',keep='any') + .select(hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('user_id')).alias('user'), + pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'), + pl.col('day_type').cast(pl.Int8).alias('dt'), + pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).list.eval(pl.element().str.slice(4,15).str.to_integer(base=16)).alias('h_items'), + pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts'))) + ev.collect(engine='streaming').sort('ts').write_parquet(f'{D}/{name}_events.parquet') + print(name,'events done',time.time()-t,flush=True) +prep('evaluation',False) +prep('train',True) diff --git a/v12/sol_work/src10/seq.py b/v12/sol_work/src10/seq.py new file mode 100644 index 0000000000000000000000000000000000000000..6b07f7ca08afbe9d5c068edce7d59cb481e12297 --- /dev/null +++ b/v12/sol_work/src10/seq.py @@ -0,0 +1,116 @@ +"""v8 sequence model (shipped config: last 500k train events < C, 3 epochs, batch 256, <=6000 steps, NP 3, no slate) (DIN-style target attention), refit per cutoff C on train events ts logit; BCE on all 10 slate rows. +Target tables only use their own history + train events < C (no aggregates over unlabelled target/eval rows). +Usage: python seq.py NAME... [--frac 0.25] -> feat_sq/NAME.parquet (rowid, sq_logit, sq_z, sq_dm, sq_lse)""" +import sys, os, time, argparse, numpy as np, polars as pl, torch, torch.nn as nn +D = os.environ.get('SQ_WORK', '@@WORK@@') +sys.path.insert(0, f'{D}/v3') +from features import load, click_log +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +from split import J20, DAY, first_events +ap = argparse.ArgumentParser(); ap.add_argument('names', nargs='+'); ap.add_argument('--frac', type=float, default=1.0) +ap.add_argument('--ntr', type=int, default=int(os.environ.get('SQ_NTR', 500000))); ap.add_argument('--ep', type=int, default=int(os.environ.get('SQ_EP', 3))) +ap.add_argument('--d', type=int, default=32); ap.add_argument('--out', default=os.environ.get('SQ_OUT', f'{D}/feat_sq')) +a = ap.parse_args(); os.makedirs(a.out, exist_ok=True) +NT = int(os.environ.get('SQ_THREADS', 4)); torch.set_num_threads(NT); torch.manual_seed(0); np.random.seed(0) +DATA = f'{D}/v3/data' +TE, TR, EE, ER, NI, NU = load(DATA) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val23': (J20 + 3 * DAY, None), 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval')} +HL, NP = 20, int(os.environ.get('SQ_NP', 3)); SLATE = int(os.environ.get('SQ_SLATE', 0)) +def slates(EV, RW): + """events EV (eid,u,ts,tb,dt,h,h_ts) + rows -> arrays in EV order; candidate matrix [E,10], clicks, rowids.""" + R = RW.join(EV.select('eid', pl.int_range(pl.len()).alias('_e')), on='eid', how='inner').sort('_e', 'rowid') + e = R['_e'].to_numpy(); assert (np.bincount(e, minlength=EV.height) == 10).all() + cand = R['ii'].to_numpy().reshape(-1, 10).astype(np.int64) + clk = R['clicked'].to_numpy().reshape(-1, 10).astype(np.float32) if 'clicked' in R.columns else None + return cand, clk, R['rowid'].to_numpy().reshape(-1, 10) +# global train index sorted by (u, ts, eid): used for prior impressions +TS = TE.sort('u', 'ts', 'eid'); T_u = TS['u'].to_numpy().astype(np.int64); T_ts = TS['ts'].to_numpy() +T_cand, T_clk, _ = slates(TS, TR); KEY = T_u * (1 << 32) + T_ts +def tokens(EV, C): + """[E, HL+NP*10] item / flag / gap-bucket tokens + ctx ints.""" + n = EV.height; u = EV['u'].to_numpy().astype(np.int64); ts = EV['ts'].to_numpy() + it = np.full((n, HL + NP * 10), NI, np.int64); fl = np.zeros_like(it); gp = np.zeros_like(it) + hh = EV['h'].to_list(); ht = EV['h_ts'].to_list() + for i in range(n): + l = hh[i] + if l: + l = l[-HL:]; t = np.asarray(ht[i][-HL:]); k = len(l) + it[i, :k] = l; fl[i, :k] = 1; gp[i, :k] = 1 + np.clip(np.log2(np.maximum(ts[i] - t, 60) / 60), 0, 19).astype(np.int64) + lim = np.minimum(ts, C); pos = np.searchsorted(KEY, u * (1 << 32) + lim, 'left') - 1 # last train event with ts < lim + npr = np.zeros(n, np.int64) + for j in range(NP): + p = pos - j; ok = (p >= 0) & (T_u[np.maximum(p, 0)] == u); p = np.maximum(p, 0) + assert (T_ts[p][ok] < lim[ok]).all() and (T_ts[p][ok] < C).all() + s = slice(HL + 10 * j, HL + 10 * (j + 1)) + it[:, s] = np.where(ok[:, None], T_cand[p], NI); fl[:, s] = np.where(ok[:, None], 2 + T_clk[p].astype(np.int64), 0) + g = 1 + np.clip(np.log2(np.maximum(ts - T_ts[p], 60) / 60), 0, 19).astype(np.int64); gp[:, s] = np.where(ok, g, 0)[:, None] + npr += ok + ctx = np.stack([EV['tb'].to_numpy().astype(np.int64), EV['dt'].to_numpy().astype(np.int64), (fl[:, :HL] > 0).sum(1), npr], 1) + return it, fl, gp, ctx +def ppmi_E(C, k=64): + P_ev = TE.filter(pl.col('ts') < C); P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CLg = click_log(C, P_ev, P_rows, P_ev.head(0)); assert CLg['ts'].max() < C + s_ = CLg.select('u', pl.col('ii').cast(pl.Int64)).to_numpy(); u, it = s_[:, 0], s_[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32) + for d in range(1, 6): + m = u[d:] == u[:-d]; x, y = it[:-d][m], it[d:][m]; ok = x != y; x, y = x[ok], y[ok]; w = np.full(len(x), 1.0 / d, np.float32) + A = A + sparse.csr_matrix((w, (x, y)), shape=(NI, NI)) + sparse.csr_matrix((w, (y, x)), shape=(NI, NI)) + A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum() + pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0 + Mx = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI)) + U, S, _ = randomized_svd(Mx, k, random_state=0, n_iter=5); V = U * np.sqrt(S); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9) + return np.vstack([V, np.zeros((1, k))]).astype(np.float32) +class M(nn.Module): + def __init__(s, EMB, d=32, H=2, dh=16): + super().__init__(); s.H, s.dh = H, dh + s.E0 = nn.Embedding.from_pretrained(torch.from_numpy(EMB), freeze=True); s.P = nn.Linear(EMB.shape[1], d); s.S = nn.Embedding(2, H) + s.E = lambda x: s.P(s.E0(x)); s.F = nn.Embedding(4, d); s.G = nn.Embedding(21, d); + s.c = nn.ModuleList([nn.Embedding(4, 4), nn.Embedding(2, 2), nn.Embedding(HL + 1, 4), nn.Embedding(NP + 1, 2)]) + s.q = nn.Linear(d, H * dh); s.k = nn.Linear(d, H * dh); s.v = nn.Linear(d, H * dh); s.null = nn.Parameter(torch.zeros(1, 1, H, dh)) + s.mlp = nn.Sequential(nn.Linear(d * (1 + 2 * SLATE) + H * dh * (1 + SLATE) + 12, 64), nn.ReLU(), nn.Linear(64, 1)) + def forward(s, it, fl, gp, ctx, cand): + B, T = it.shape; K = s.E(it) + s.F(fl) + s.G(gp); q = s.E(cand) # (B,T,d), (B,10,d) + Q = s.q(q).view(B, 10, s.H, s.dh); Kk = s.k(K).view(B, T, s.H, s.dh); V = s.v(K).view(B, T, s.H, s.dh) + same = (cand[:, :, None] == it[:, None, :]).long(); att = torch.einsum('bchd,bthd->bcht', Q, Kk) / s.dh ** 0.5 + s.S(same).permute(0, 1, 3, 2); att = att.masked_fill((fl == 0)[:, None, None, :], -1e9) + an = (Q * s.null).sum(-1, keepdim=True) / s.dh ** 0.5; w = torch.softmax(torch.cat([att, an], -1), -1) + P = torch.einsum('bcht,bthd->bchd', w[..., :T], V).reshape(B, 10, -1) + cx = torch.cat([e(ctx[:, i]) for i, e in enumerate(s.c)], -1)[:, None].expand(B, 10, 12) + f = [q, P, cx] + if SLATE: # within-event context: the slate's own candidates only + qm = q.mean(1, keepdim=True).expand_as(q); Pm = P.mean(1, keepdim=True).expand_as(P); f = [q, q - qm, qm, P, P - Pm, cx] + return s.mlp(torch.cat(f, -1)).squeeze(-1) +def usel(u): return (u * 2654435761 % 2 ** 32) / 2 ** 32 < a.frac +_M = {} +def fit(C): + if C in _M: return _M[C] + t0 = time.time(); m = T_ts < C; idx = np.nonzero(m)[0]; idx = idx[np.argsort(T_ts[idx], kind='stable')][-a.ntr:] # train on all users; --frac only subsamples target users + it, fl, gp, ctx = tokens(TS[idx], C); cand = T_cand[idx]; y = T_clk[idx]; tk = time.time() - t0 + X = [torch.from_numpy(z) for z in (it, fl, gp, ctx, cand)]; Y = torch.from_numpy(y) + te = time.time(); EMB = ppmi_E(C); print(f' ppmi C={C} {time.time() - te:.0f}s', flush=True) + mdl = M(EMB, a.d); opt = torch.optim.Adam(mdl.parameters(), 2e-3); lossf = nn.BCEWithLogitsLoss(); bs = int(os.environ.get('SQ_BS', 256)); n = len(idx) + maxst = int(os.environ.get('SQ_MAXSTEPS', 6000)); st = 0 # v8: hard cap on optimizer steps -> bounded eval-time fit + for ep in range(a.ep): + perm = torch.randperm(n); tl = 0 + for b in range(0, n, bs): + if st >= maxst: break + st += 1; j = perm[b:b + bs]; opt.zero_grad(); l = lossf(mdl(*[x[j] for x in X]), Y[j]); l.backward(); opt.step(); tl += l.item() * len(j) + print(f' C={C} ep{ep} loss {tl / n:.4f} n={n} steps={st} {time.time() - t0:.0f}s', flush=True) + mdl.eval(); _M[C] = mdl; print(f' fit C={C} tok {tk:.0f}s total {time.time() - t0:.0f}s', flush=True); return mdl +def score(mdl, EV, C, cand): + it, fl, gp, ctx = tokens(EV, C); out = [] + with torch.no_grad(): + for b in range(0, len(cand), 8192): + out.append(mdl(*[torch.from_numpy(z[b:b + 8192]) for z in (it, fl, gp, ctx, cand)]).numpy()) + return np.concatenate(out) +for name in a.names: + t0 = time.time(); C, W = SPEC[name] + if W == 'eval': EV, RW = EE, ER + else: EV = first_events(TE, C, W); RW = TR.join(EV.select('eid'), on='eid', how='semi') + if a.frac < 1: EV = EV.filter(pl.Series(usel(EV['u'].to_numpy().astype(np.int64)))) + cand, _, rid = slates(EV, RW); S = score(fit(C), EV, C, cand) + z = (S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6); lse = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1) + df = pl.DataFrame({'rowid': rid.ravel(), 'sq_logit': S.ravel(), 'sq_z': z.ravel(), 'sq_dm': (S - S.mean(1, keepdims=True)).ravel(), 'sq_lse': lse.ravel()}).with_columns(pl.exclude('rowid').cast(pl.Float32)) + df.write_parquet(f'{a.out}/{name}.parquet'); print(name, df.shape, f'{time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/seq10.py b/v12/sol_work/src10/seq10.py new file mode 100644 index 0000000000000000000000000000000000000000..b7e39f7059f865482653f6aad6b79f2b11304305 --- /dev/null +++ b/v12/sol_work/src10/seq10.py @@ -0,0 +1,147 @@ +"""v10 seq10.py = copy of shipped v8 seq.py + extra outputs for LightGBM stacking (attention stats, user/candidate dot+cos, within-event rank/gap); +fit() reseeds torch per cutoff so each fit is order-independent. Encoder variant via SQ_ENC (din|gru|rec). +v8 sequence model (shipped config: last 500k train events < C, 3 epochs, batch 256, <=6000 steps, NP 3, no slate) (DIN-style target attention), refit per cutoff C on train events ts logit; BCE on all 10 slate rows. +Target tables only use their own history + train events < C (no aggregates over unlabelled target/eval rows). +Usage: python seq.py NAME... [--frac 0.25] -> feat_sq/NAME.parquet (rowid, sq_logit, sq_z, sq_dm, sq_lse)""" +import sys, os, time, argparse, numpy as np, polars as pl, torch, torch.nn as nn +D = os.environ.get('SQ_WORK', '@@WORK@@') +sys.path.insert(0, f'{D}/v3') +from features import load, click_log +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +from split import J20, DAY, first_events +ap = argparse.ArgumentParser(); ap.add_argument('names', nargs='+'); ap.add_argument('--frac', type=float, default=1.0) +ap.add_argument('--ntr', type=int, default=int(os.environ.get('SQ_NTR', 500000))); ap.add_argument('--ep', type=int, default=int(os.environ.get('SQ_EP', 3))) +ap.add_argument('--d', type=int, default=32); ap.add_argument('--out', default=os.environ.get('SQ_OUT', f'{D}/feat_sq10')) +a = ap.parse_args(); os.makedirs(a.out, exist_ok=True) +NT = int(os.environ.get('SQ_THREADS', 4)); torch.set_num_threads(NT); torch.manual_seed(0); np.random.seed(0) +DATA = f'{D}/v3/data' +TE, TR, EE, ER, NI, NU = load(DATA) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val23': (J20 + 3 * DAY, None), 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval')} +ENC = os.environ.get('SQ_ENC', 'din'); HL, NP = 20, int(os.environ.get('SQ_NP', 3)); SLATE = int(os.environ.get('SQ_SLATE', 0)) +def slates(EV, RW): + """events EV (eid,u,ts,tb,dt,h,h_ts) + rows -> arrays in EV order; candidate matrix [E,10], clicks, rowids.""" + R = RW.join(EV.select('eid', pl.int_range(pl.len()).alias('_e')), on='eid', how='inner').sort('_e', 'rowid') + e = R['_e'].to_numpy(); assert (np.bincount(e, minlength=EV.height) == 10).all() + cand = R['ii'].to_numpy().reshape(-1, 10).astype(np.int64) + clk = R['clicked'].to_numpy().reshape(-1, 10).astype(np.float32) if 'clicked' in R.columns else None + return cand, clk, R['rowid'].to_numpy().reshape(-1, 10) +# global train index sorted by (u, ts, eid): used for prior impressions +TS = TE.sort('u', 'ts', 'eid'); T_u = TS['u'].to_numpy().astype(np.int64); T_ts = TS['ts'].to_numpy() +T_cand, T_clk, _ = slates(TS, TR); KEY = T_u * (1 << 32) + T_ts +def tokens(EV, C): + """[E, HL+NP*10] item / flag / gap-bucket tokens + ctx ints.""" + n = EV.height; u = EV['u'].to_numpy().astype(np.int64); ts = EV['ts'].to_numpy() + it = np.full((n, HL + NP * 10), NI, np.int64); fl = np.zeros_like(it); gp = np.zeros_like(it) + hh = EV['h'].to_list(); ht = EV['h_ts'].to_list() + for i in range(n): + l = hh[i] + if l: + l = l[-HL:]; t = np.asarray(ht[i][-HL:]); k = len(l) + it[i, :k] = l; fl[i, :k] = 1; gp[i, :k] = 1 + np.clip(np.log2(np.maximum(ts[i] - t, 60) / 60), 0, 19).astype(np.int64) + lim = np.minimum(ts, C); pos = np.searchsorted(KEY, u * (1 << 32) + lim, 'left') - 1 # last train event with ts < lim + npr = np.zeros(n, np.int64) + for j in range(NP): + p = pos - j; ok = (p >= 0) & (T_u[np.maximum(p, 0)] == u); p = np.maximum(p, 0) + assert (T_ts[p][ok] < lim[ok]).all() and (T_ts[p][ok] < C).all() + s = slice(HL + 10 * j, HL + 10 * (j + 1)) + it[:, s] = np.where(ok[:, None], T_cand[p], NI); fl[:, s] = np.where(ok[:, None], 2 + T_clk[p].astype(np.int64), 0) + g = 1 + np.clip(np.log2(np.maximum(ts - T_ts[p], 60) / 60), 0, 19).astype(np.int64); gp[:, s] = np.where(ok, g, 0)[:, None] + npr += ok + ctx = np.stack([EV['tb'].to_numpy().astype(np.int64), EV['dt'].to_numpy().astype(np.int64), (fl[:, :HL] > 0).sum(1), npr], 1) + return it, fl, gp, ctx +def ppmi_E(C, k=64): + P_ev = TE.filter(pl.col('ts') < C); P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CLg = click_log(C, P_ev, P_rows, P_ev.head(0)); assert CLg['ts'].max() < C + s_ = CLg.select('u', pl.col('ii').cast(pl.Int64)).to_numpy(); u, it = s_[:, 0], s_[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32) + for d in range(1, 6): + m = u[d:] == u[:-d]; x, y = it[:-d][m], it[d:][m]; ok = x != y; x, y = x[ok], y[ok]; w = np.full(len(x), 1.0 / d, np.float32) + A = A + sparse.csr_matrix((w, (x, y)), shape=(NI, NI)) + sparse.csr_matrix((w, (y, x)), shape=(NI, NI)) + A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum() + pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0 + Mx = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI)) + U, S, _ = randomized_svd(Mx, k, random_state=0, n_iter=5); V = U * np.sqrt(S); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9) + return np.vstack([V, np.zeros((1, k))]).astype(np.float32) +class M(nn.Module): + def __init__(s, EMB, d=32, H=2, dh=16): + super().__init__(); s.H, s.dh = H, dh + s.E0 = nn.Embedding.from_pretrained(torch.from_numpy(EMB), freeze=True); s.P = nn.Linear(EMB.shape[1], d); s.S = nn.Embedding(2, H) + s.E = lambda x: s.P(s.E0(x)); s.F = nn.Embedding(4, d); s.G = nn.Embedding(21, d); + s.c = nn.ModuleList([nn.Embedding(4, 4), nn.Embedding(2, 2), nn.Embedding(HL + 1, 4), nn.Embedding(NP + 1, 2)]) + s.q = nn.Linear(d, H * dh); s.k = nn.Linear(d, H * dh); s.v = nn.Linear(d, H * dh); s.null = nn.Parameter(torch.zeros(1, 1, H, dh)) + if ENC == 'rec': s.R = nn.Embedding(21, d) + if ENC == 'gru': s.gru = nn.GRU(d, d, batch_first=True) + s.mlp = nn.Sequential(nn.Linear(d * (1 + 2 * SLATE) + H * dh * (1 + SLATE) + 12, 64), nn.ReLU(), nn.Linear(64, 1)) + def forward(s, it, fl, gp, ctx, cand, extra=False): + B, T = it.shape; K = s.E(it) + s.F(fl) + s.G(gp); q = s.E(cand) # (B,T,d), (B,10,d) + if ENC == 'rec': K = K + s.R(gp) # recency-aware keys (extra gap emb) + if ENC == 'gru': # GRU over tokens in reverse-recency order -> contextualised keys + K = K + s.gru(K)[0] + Q = s.q(q).view(B, 10, s.H, s.dh); Kk = s.k(K).view(B, T, s.H, s.dh); V = s.v(K).view(B, T, s.H, s.dh) + same = (cand[:, :, None] == it[:, None, :]).long(); att = torch.einsum('bchd,bthd->bcht', Q, Kk) / s.dh ** 0.5 + s.S(same).permute(0, 1, 3, 2); att = att.masked_fill((fl == 0)[:, None, None, :], -1e9) + an = (Q * s.null).sum(-1, keepdim=True) / s.dh ** 0.5; w = torch.softmax(torch.cat([att, an], -1), -1) + P = torch.einsum('bcht,bthd->bchd', w[..., :T], V).reshape(B, 10, -1) + cx = torch.cat([e(ctx[:, i]) for i, e in enumerate(s.c)], -1)[:, None].expand(B, 10, 12) + f = [q, P, cx] + if SLATE: # within-event context: the slate's own candidates only + qm = q.mean(1, keepdim=True).expand_as(q); Pm = P.mean(1, keepdim=True).expand_as(P); f = [q, q - qm, qm, P, P - Pm, cx] + out = s.mlp(torch.cat(f, -1)).squeeze(-1) + if not extra: return out + valid = (fl > 0).float(); nv = valid.sum(1).clamp(min=1) # (B,T),(B,) + wm = w.mean(2) # head-mean weights (B,10,T+1) + a_max = wm[..., :T].max(-1).values; a_null = wm[..., T]; a_mean = (1 - a_null) / nv[:, None] + ent = -(wm.clamp(min=1e-9).log() * wm).sum(-1) + vq = s.v(q) # candidate in value space (B,10,H*dh) + pdot = (P * vq).sum(-1); pcos = pdot / (P.norm(dim=-1) * vq.norm(dim=-1) + 1e-6) + mu = (K * valid[..., None]).sum(1) / nv[:, None] # mean-pooled user emb (B,d) + mdot = (q * mu[:, None]).sum(-1); mcos = mdot / (q.norm(dim=-1) * mu.norm(dim=-1)[:, None] + 1e-6) + ck = ((fl == 1) | (fl == 3)).float(); e0 = s.E0(it); eu = (e0 * ck[..., None]).sum(1); ec = s.E0(cand) + ecos = (ec * eu[:, None]).sum(-1) / (ec.norm(dim=-1) * eu.norm(dim=-1)[:, None] + 1e-6) + return out, torch.stack([a_max, a_null, a_mean, ent, pdot, pcos, mdot, mcos, ecos], -1) +XN = ['sq_amax', 'sq_anull', 'sq_amean', 'sq_aent', 'sq_pdot', 'sq_pcos', 'sq_mdot', 'sq_mcos', 'sq_ecos'] +def usel(u): return (u * 2654435761 % 2 ** 32) / 2 ** 32 < a.frac +_M = {} +def fit(C): + if C in _M: return _M[C] + torch.manual_seed(0); np.random.seed(0) + t0 = time.time(); m = T_ts < C; idx = np.nonzero(m)[0]; idx = idx[np.argsort(T_ts[idx], kind='stable')][-a.ntr:] # train on all users; --frac only subsamples target users + it, fl, gp, ctx = tokens(TS[idx], C); cand = T_cand[idx]; y = T_clk[idx]; tk = time.time() - t0 + X = [torch.from_numpy(z) for z in (it, fl, gp, ctx, cand)]; Y = torch.from_numpy(y) + te = time.time(); EMB = ppmi_E(C); print(f' ppmi C={C} {time.time() - te:.0f}s', flush=True) + mdl = M(EMB, a.d); opt = torch.optim.Adam(mdl.parameters(), float(os.environ.get('SQ_LR', 2e-3))); lossf = nn.BCEWithLogitsLoss(); bs = int(os.environ.get('SQ_BS', 256)); n = len(idx) + maxst = int(os.environ.get('SQ_MAXSTEPS', 6000)); st = 0 # v8: hard cap on optimizer steps -> bounded eval-time fit + for ep in range(a.ep): + perm = torch.randperm(n); tl = 0 + for b in range(0, n, bs): + if st >= maxst: break + st += 1; j = perm[b:b + bs]; opt.zero_grad(); l = lossf(mdl(*[x[j] for x in X]), Y[j]); l.backward(); opt.step(); tl += l.item() * len(j) + print(f' C={C} ep{ep} loss {tl / n:.4f} n={n} steps={st} {time.time() - t0:.0f}s', flush=True) + mdl.eval(); _M[C] = mdl; print(f' fit C={C} tok {tk:.0f}s total {time.time() - t0:.0f}s', flush=True); return mdl +def score(mdl, EV, C, cand): + it, fl, gp, ctx = tokens(EV, C); out = []; ex = [] + with torch.no_grad(): + for b in range(0, len(cand), 8192): + o, e = mdl(*[torch.from_numpy(z[b:b + 8192]) for z in (it, fl, gp, ctx, cand)], extra=True); out.append(o.numpy()); ex.append(e.numpy()) + return np.concatenate(out), np.concatenate(ex) +TIM = {} +for name in a.names: + t0 = time.time(); name, _, mn = name.partition('@'); C, W = SPEC[name]; CM = SPEC[mn][0] if mn else C # 'f4@f3': score f4 rows (history at C_f4) with the model fit at C_f3 < C_f4 + assert CM <= C + if W == 'eval': EV, RW = EE, ER + else: EV = first_events(TE, C, W); RW = TR.join(EV.select('eid'), on='eid', how='semi') + if a.frac < 1: EV = EV.filter(pl.Series(usel(EV['u'].to_numpy().astype(np.int64)))) + cand, _, rid = slates(EV, RW); S, X = score(fit(CM), EV, C, cand) + z = (S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6); lse = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1) + rk = (np.argsort(np.argsort(-S, 1, kind='stable'), 1) + 1).astype(np.float32) # 1 = event top + gap = S - S.max(1, keepdims=True); gap2 = S - np.sort(S, 1)[:, -2:-1] # gap to top / to runner-up + cols = {'rowid': rid.ravel(), 'sq_logit': S.ravel(), 'sq_z': z.ravel(), 'sq_dm': (S - S.mean(1, keepdims=True)).ravel(), 'sq_lse': lse.ravel(), + 'sq_rank': rk.ravel(), 'sq_gap': gap.ravel(), 'sq_gap2': gap2.ravel()} + for i, n_ in enumerate(XN): cols[n_] = X[..., i].ravel() + for n_ in ('sq_pcos', 'sq_mcos', 'sq_ecos'): # within-event demeaned versions + v = X[..., XN.index(n_)]; cols[n_ + '_dm'] = (v - v.mean(1, keepdims=True)).ravel() + for n_ in ['sq_pdot', 'sq_pcos', 'sq_mdot', 'sq_mcos']: cols.pop(n_) # v10B: fit-specific scale drifts train->eval (KS .09-.16); keep _dm versions + df = pl.DataFrame(cols).with_columns(pl.exclude('rowid').cast(pl.Float32)) + df.write_parquet(f'{a.out}/{name}.parquet'); print(name, df.shape, f'{time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/ttfeat.py b/v12/sol_work/src10/ttfeat.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1e9bdb817c6edf397fb30603cc806edbefa529 --- /dev/null +++ b/v12/sol_work/src10/ttfeat.py @@ -0,0 +1,82 @@ +"""data_dig_v2 features, refit per cutoff C on train only (events ts < C); eval fitted at J27 on all train. +(A) row-scaled exact SVD: binary user x item of ALL pre-C history clicks, diag(1/sqrt(len_u)) M diag(1/sqrt(pop+5)), svds k16 rs0, + comps 2..13 * s -> tA_cos = cos(candidate, own-history centroid) (+ _z, _dmax within event). +(B) two-tower conditional logit, tied item vectors initialised from L2-normalised (A) rows, u = mean of last<=20 own-history vectors, + score u.v, 10-way softmax CE on clicked train events (ts 0 + cand = torch.from_numpy(np.array(G['ii'].to_list(), np.int64)[keep]); lab = torch.from_numpy(clk[keep].argmax(1).astype(np.int64)) + Ht = torch.from_numpy(hmat(G.filter(pl.Series(keep)))) + torch.manual_seed(0); np.random.seed(0) + P = torch.nn.Embedding(NI + 1, 12, padding_idx=NI) + with torch.no_grad(): P.weight[:NI] = torch.from_numpy(nrm(VA)); P.weight[NI] = 0 + opt = torch.optim.Adam(P.parameters(), lr=1e-3); N = len(lab) + def uvec(H): + m = (H < NI).float(); return (P(H) * m[..., None]).sum(1) / m.sum(1, keepdim=True).clamp(min=1) + for ep in range(6): + perm = torch.randperm(N); tot = 0.0 + for i in range(0, N, 2048): + j = perm[i:i + 2048]; sc = (uvec(Ht[j])[:, None, :] * P(cand[j])).sum(-1) + loss = torch.nn.functional.cross_entropy(sc, lab[j]); opt.zero_grad(); loss.backward(); opt.step() + with torch.no_grad(): P.weight[NI] = 0 + tot += loss.item() * len(j) + print(f' C {C} ep{ep} loss {tot / N:.4f}', flush=True) + VB = P.weight.detach().numpy().copy() + print(time.strftime('%T'), 'C', C, 'train events', N, f'{time.time() - t0:.0f}s', flush=True) + _M[C] = (VA, VB); return _M[C] +def build(C, T_ev, T_rows): + VA, VB = fit(C) + E = T_ev.select('eid', 'h').with_row_index('k') + R = T_rows.select('rowid', 'eid', 'ii').join(E.select('eid', 'k'), on='eid', how='left') + rk = R['k'].to_numpy(); ri = R['ii'].to_numpy().astype(np.int64) + # (A) all-history centroid cosine + V = nrm(VA); H = E.select('k', 'h').explode('h').drop_nulls('h'); hc = np.asarray(sp.csr_matrix((np.ones(H.height), (H['k'].to_numpy(), H['h'].to_numpy().astype(np.int64))), shape=(E.height, V.shape[0])) @ V) # v8: replaces np.add.at + hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1) + a = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan) + # (B) tower dot with u = mean of last <= 20 own-history vectors + Hm = hmat(E); m = (Hm < NI); Vp = np.vstack([VB[:NI], np.zeros((1, VB.shape[1]), VB.dtype)]) + u = (Vp[Hm] * m[..., None]).sum(1) / np.maximum(m.sum(1, keepdims=True), 1) + dot = np.where(m.sum(1)[rk] > 0, (u[rk] * VB[ri]).sum(1), np.nan) + R = R.select('rowid', 'eid').with_columns(pl.Series('tA_cos', a.astype(np.float32)).fill_nan(None), pl.Series('tB_dot', dot.astype(np.float32)).fill_nan(None)) + R = R.with_columns(((pl.col('tA_cos') - pl.col('tA_cos').mean().over('eid')) / (pl.col('tA_cos').std().over('eid') + 1e-6)).alias('tA_z'), + (pl.col('tA_cos') - pl.col('tA_cos').max().over('eid')).alias('tA_dmax'), + pl.col('tB_dot').exp().sum().over('eid').log().alias('tB_lse'), pl.col('tB_dot').mean().over('eid').alias('tB_mean'), + (pl.col('tB_dot') - pl.col('tB_dot').mean().over('eid')).alias('tB_dmean'), + ((pl.col('tB_dot') - pl.col('tB_dot').mean().over('eid')) / (pl.col('tB_dot').std().over('eid') + 1e-6)).alias('tB_dot_z')) + return R.drop('eid') +for n in sys.argv[1:]: + t0 = time.time() + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{n}.parquet'); print(n, R.shape, f'{time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src10/v3/features.py b/v12/sol_work/src10/v3/features.py new file mode 100644 index 0000000000000000000000000000000000000000..c1dac35adfde2286a866d8ea1d6c94a1a5e2fe10 --- /dev/null +++ b/v12/sol_work/src10/v3/features.py @@ -0,0 +1,285 @@ +"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR. + +build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate. +Every statistic uses ONLY: + * train events/rows with ts < C (fit window: impressions + labels), and + * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts). +Since target events are each user's first event at/after C, their histories contain only clicks < C, +so nothing observed after a target row's own event time is ever joined. + +Feature groups (column prefixes, used for ablation): + hist_ candidate vs the user's click log / own history / past impressions of that user-item pair + it_ item popularity & CTR (full fit window, last 24h, last 6h, trend) + co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks + us_ user activity / CTR / recency / cold-start flags + ev_ within-event rank / z-score / event max & mean of the main features + tm_ time_bucket, day_type, hour, staleness since cutoff + sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines + mf_ logistic matrix factorisation score fitted on fit-window impressions +""" +import time +import numpy as np +import polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +J20 = 1737331200 +DAY = 86400 +G_PRIOR = 0.0629 +K_ITEM = 20 # pseudo-impressions for item CTR shrinkage +K_USER = 5 # pseudo-events for user CTR shrinkage +TOPK = 200 # co-click neighbours kept per item +ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord'] +_t0 = time.time() + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/ev_train.parquet') + TR = pl.read_parquet(f'{D}/rows_train.parquet') + EE = pl.read_parquet(f'{D}/ev_eval.parquet') + ER = pl.read_parquet(f'{D}/rows_eval.parquet') + NI = pl.read_parquet(f'{D}/items.parquet').height + NU = pl.read_parquet(f'{D}/users.parquet').height + return TE, TR, EE, ER, NI, NU + +def _hist_long(E): + return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts'))) + +def click_log(C, P_ev, P_rows, T_ev): + """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories.""" + win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts') + hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C) + CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii') + return CL + +def cooc_tables(CL, NI): + """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next.""" + us = CL.select('u', 'ii').unique() + pop = us.group_by('ii').agg(pl.len().alias('pop')) + pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b')) + .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co'))) + co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b') + .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos')) + .select(pl.col('ii_b').alias('j'), 'ii', 'cos') # j = history item, ii = candidate + .sort('cos', descending=True).group_by('j').head(TOPK)) + seq = CL.select('u', 'ii') # already sorted by u, ts + parts = [] + for lag in (1, 2, 3): + parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j') + .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum()) + tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot')) + trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp')) + .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK)) + return co, trn + +def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000): + """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy).""" + rng = np.random.default_rng(seed) + u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64) + y = P_rows['clicked'].to_numpy().astype(np.float32) + P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32) + bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32) + b = np.float32(np.log(y.mean() / (1 - y.mean()))) + gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6) + n = len(y) + for ep in range(epochs): + perm = rng.permutation(n) + for s in range(0, n, batch): + idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx] + pu, qi = P[ub], Q[ib] + z = b + bu[ub] + bi[ib] + (pu * qi).sum(1) + g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz + m = len(idx) + Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m)) + Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m)) + dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None] + dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None] + dbu = Su @ g; dbi = Si @ g + gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi + P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ) + bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi) + return dict(b=b, bu=bu, bi=bi, P=P, Q=Q) + +def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + g = float(P_rows['clicked'].mean()) + ndays = (C - J20) / DAY + CL = click_log(C, P_ev, P_rows, T_ev) + + # ---------------- item ---------------- + def ictr(c, n): + return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM) + it = P_rows.group_by('ii').agg( + pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'), + (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'), + (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6')) + it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'), + ictr('it_clk6', 'it_imp6').alias('it_ctr6')) + it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'), + (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'), + (pl.col('it_imp') / ndays).alias('it_imp_rate')) + ip = CL.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'), + (pl.col('ts') < J20).sum().alias('it_nclk_warm'), + ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d')) + + # ---------------- user ---------------- + ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'), + pl.col('ts').max().alias('_u_last_ev')) + ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr')) + uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'), + pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'), + (pl.col('ts') >= C - DAY).sum().alias('us_nclk24')) + uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate')) + + # ---------------- user x item ---------------- + uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'), + (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk')) + uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'), + pl.col('ts').max().alias('_ui_last_imp')) + uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk')) + + # ---------------- assemble ---------------- + R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns]) + .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left') + .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left')) + R = R.with_columns( + pl.col('h').list.len().alias('us_hlen'), + (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'), + pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'), + (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'), + (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'), + (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'), + (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'), + (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'), + pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt')) + # position (from end, 1 = most recent) of candidate in own history + hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii')) + .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own'))) + R = R.join(hx, on='r', how='left') + R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY), + pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY)) + R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in')) + + # ---------------- co-click ---------------- + co, trn = cooc_tables(CL, NI) + last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20')) + T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left') + X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos']) + .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32)) + .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left') + .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0))) + l5 = pl.col('pos') <= 5 + agg = X.group_by('r').agg( + pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'), + pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'), + pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'), + pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'), + pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'), + pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'), + (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20')) + R = R.join(agg, on='r', how='left').sort('r') + + # ---------------- SVD embeddings ---------------- + # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories; + # 'lag1d' = only clicks/impressions with ts < C - 1 day + if sv_mode == 'std': + CLsv, NEGsrc = CL, P_rows + elif sv_mode == 'notgt': + CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'), + _hist_long(P_ev).filter(pl.col('ts') < C)]).unique() + NEGsrc = P_rows + elif sv_mode == 'lag1d': + CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY) + else: + raise ValueError(sv_mode) + assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C + cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc + uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64) + ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos']) + # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway + er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32) + er_l5 = ex['pos'].to_numpy() <= 5 + nR = R.height + svcols = {} + for k in (16, 32, 64): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = (U * np.sqrt(S)).astype(np.float32) + Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9) + cand = Vn[iic] + svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1) + svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1) + hv = np.zeros((nR, k), np.float32); np.add.at(hv, er_r, Vn[er_j] * er_w[:, None]) + hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9) + if k == 32: + svcols['sv_k32_hnorm'] = hn + sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1) + mx = np.full(nR, -1.0, np.float32); np.maximum.at(mx, er_r, sims) + mx5 = np.full(nR, -1.0, np.float32); np.maximum.at(mx5, er_r[er_l5], sims[er_l5]) + svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5 + if k == 64: + # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only) + Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + neg = NEGsrc.filter(pl.col('clicked') == 0) + Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI)) + nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9 + nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9 + cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1) + svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn + for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')): + M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9 + svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()]) + + # ---------------- MF ---------------- + if with_mf: + mf = fit_mf(P_rows, NU, NI) + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + dot = (mf['P'][uu] * mf['Q'][ii]).sum(1) + R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot), + pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii])) + + fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] + + [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] + + [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)]) + + # ---------------- event context ---------------- + main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk', + 'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last', + 'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else []) + exprs = [] + for c in main: + x = pl.col(c).cast(pl.Float64) + exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'), + ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'), + x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')] + R = R.with_columns(exprs) + R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r') + log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s') + return R + +GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf'] + +def feature_cols(df, groups): + cols = [] + for c in df.columns: + if c in ID_COLS: continue + gname = c.split('_')[0] + if gname == 'ev': + src = c[3:].split('_')[0] # ev__... + if 'ev' in groups and src in groups: cols.append(c) + elif gname in groups: + cols.append(c) + return cols diff --git a/v12/sol_work/src10/v3/prep.py b/v12/sol_work/src10/v3/prep.py new file mode 100644 index 0000000000000000000000000000000000000000..d765f00826f7df7a80e886852b2c2571938b4eb6 --- /dev/null +++ b/v12/sol_work/src10/v3/prep.py @@ -0,0 +1,65 @@ +"""Parse train/evaluation CSVs into compact parquet tables (dense int ids). + +Outputs in v3/data/: + rows_train.parquet : rowid, eid, ii (item idx), clicked + rows_eval.parquet : rowid, eid, ii, ord (original row order) + ev_train.parquet / ev_eval.parquet : eid, u (user idx), ts, tb, dt, h (list[int item idx]), h_ts (list[int]) + items.parquet / users.parquet : id string -> idx +Usage: python prep.py [DATA_DIR] +""" +import polars as pl, sys, time, os +DATA = sys.argv[1] if len(sys.argv) > 1 else '@@RAW@@' +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') +os.makedirs(OUT, exist_ok=True) +t0 = time.time() +def log(*a): print(f'[{time.time()-t0:6.1f}s]', *a, flush=True) + +def scan(name): + return pl.scan_csv(f'{DATA}/{name}.csv', schema_overrides={'history_item_ids': pl.Utf8, 'history_timestamps': pl.Utf8, + 'rowid': pl.Utf8, 'event_id': pl.Utf8, 'user_id': pl.Utf8, 'item_id': pl.Utf8}) + +raw = {} +for name, lab in (('train', True), ('evaluation', False)): + lf = scan(name) + cols = ['rowid', 'event_id', 'item_id'] + (['clicked'] if lab else []) + rows = lf.select(cols).collect(engine='streaming') + ev = (lf.select('event_id', 'user_id', 'timestamp', 'time_bucket', 'day_type', 'history_item_ids', 'history_timestamps') + .unique(subset='event_id', keep='any') + .with_columns(pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).alias('h_str'), + pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts')) + .drop('history_item_ids', 'history_timestamps').collect(engine='streaming')) + raw[name] = (rows, ev) + log(name, rows.shape, ev.shape) + +tr_rows, tr_ev = raw['train']; ev_rows, ev_ev = raw['evaluation'] +items = pl.concat([tr_rows['item_id'], ev_rows['item_id'], tr_ev['h_str'].explode().drop_nulls(), + ev_ev['h_str'].explode().drop_nulls()]).unique().sort() +items = pl.DataFrame({'item_id': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)}) +users = pl.concat([tr_ev['user_id'], ev_ev['user_id']]).unique().sort() +users = pl.DataFrame({'user_id': users, 'u': pl.arange(0, len(users), eager=True).cast(pl.Int32)}) +events = pl.concat([tr_ev['event_id'], ev_ev['event_id']]).unique().sort() +events = pl.DataFrame({'event_id': events, 'eid': pl.arange(0, len(events), eager=True).cast(pl.Int32)}) +log('items', len(items), 'users', len(users), 'events', len(events)) +items.write_parquet(f'{OUT}/items.parquet'); users.write_parquet(f'{OUT}/users.parquet') + +def map_rows(rows, lab): + r = rows.with_row_index('ord').join(events, on='event_id', how='left').join(items, on='item_id', how='left') + sel = ['ord', 'rowid', 'eid', 'ii'] + (['clicked'] if lab else []) + r = r.sort('ord').select(sel) + if lab: r = r.with_columns(pl.col('clicked').cast(pl.Int8)) + return r + +def map_ev(ev): + e = ev.join(events, on='event_id', how='left').join(users, on='user_id', how='left') + ex = e.select('eid', 'h_str').with_columns(pl.int_ranges(0, pl.col('h_str').list.len()).alias('k')).explode(['h_str', 'k']).drop_nulls('h_str') + ex = ex.join(items.rename({'item_id': 'h_str'}), on='h_str', how='left').sort('eid', 'k') + h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').alias('h')) + e = e.join(h, on='eid', how='left').with_columns(pl.col('h').fill_null(pl.lit([], dtype=pl.List(pl.Int32)))) + return e.select('eid', 'u', pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'), + pl.col('day_type').cast(pl.Int8).alias('dt'), 'h', 'h_ts').sort('ts', 'eid') + +map_rows(tr_rows, True).drop('ord').write_parquet(f'{OUT}/rows_train.parquet') +map_rows(ev_rows, False).write_parquet(f'{OUT}/rows_eval.parquet') +map_ev(tr_ev).write_parquet(f'{OUT}/ev_train.parquet') +map_ev(ev_ev).write_parquet(f'{OUT}/ev_eval.parquet') +log('done') diff --git a/v12/sol_work/src10/v3/split.py b/v12/sol_work/src10/v3/split.py new file mode 100644 index 0000000000000000000000000000000000000000..7ec658f41a419f776d82b2bf76d97bc2ba3cd155 --- /dev/null +++ b/v12/sol_work/src10/v3/split.py @@ -0,0 +1,36 @@ +"""Time-based splits that mirror the official evaluation. + +Evaluation = each user's FIRST event at/after the cutoff. So for a cutoff C: + fit events = all train events with ts < C (labels usable for aggregates) + target/val = for each user, the first train event with C <= ts < C+W (W=None -> until end of train) +Whole events always stay together (split is by event, rows follow their event). +""" +import polars as pl + +J20 = 1737331200 # 2025-01-20 00:00 UTC (start of train window; earlier clicks = warm-up) +DAY = 86400 +C_MAIN = J20 + 5 * DAY # 2025-01-25 +C_SECOND = J20 + 6 * DAY # 2025-01-26 +C_FINAL = J20 + 7 * DAY # 2025-01-27 (official cutoff) + +def fit_events(TE: pl.DataFrame, C: int) -> pl.DataFrame: + return TE.filter(pl.col('ts') < C) + +def first_events(TE: pl.DataFrame, C: int, W: int | None = None) -> pl.DataFrame: + """First event per user in [C, C+W). Ties on ts broken by eid (deterministic).""" + m = pl.col('ts') >= C + if W is not None: + m = m & (pl.col('ts') < C + W) + return TE.filter(m).sort('ts', 'eid').group_by('u', maintain_order=True).first() + +def split(TE: pl.DataFrame, TR: pl.DataFrame, C: int, W: int | None = None): + """Return (fit_ev, fit_rows, val_ev, val_rows).""" + fe = fit_events(TE, C) + ve = first_events(TE, C, W) + fr = TR.join(fe.select('eid'), on='eid', how='semi') + vr = TR.join(ve.select('eid'), on='eid', how='semi') + return fe, fr, ve, vr + +def train_fold_cutoffs(C_val: int): + """Daily cutoffs strictly before C_val used to build model-training rows (each fold: first event/user in [Ck, Ck+1d)).""" + return [J20 + k * DAY for k in range(1, (C_val - J20) // DAY)] diff --git a/v12/sol_work/src10/v3feat_ne.py b/v12/sol_work/src10/v3feat_ne.py new file mode 100644 index 0000000000000000000000000000000000000000..c74e8e38d1294e2825fbc443990833bb59752b67 --- /dev/null +++ b/v12/sol_work/src10/v3feat_ne.py @@ -0,0 +1,298 @@ +"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR. + +build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate. +Every statistic uses ONLY: + * train events/rows with ts < C (fit window: impressions + labels), and + * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts). +Since target events are each user's first event at/after C, their histories contain only clicks < C, +so nothing observed after a target row's own event time is ever joined. + +Feature groups (column prefixes, used for ablation): + hist_ candidate vs the user's click log / own history / past impressions of that user-item pair + it_ item popularity & CTR (full fit window, last 24h, last 6h, trend) + co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks + us_ user activity / CTR / recency / cold-start flags + ev_ within-event rank / z-score / event max & mean of the main features + tm_ time_bucket, day_type, hour, staleness since cutoff + sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines + mf_ logistic matrix factorisation score fitted on fit-window impressions +""" +import time +import numpy as np +import polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +J20 = 1737331200 +DAY = 86400 +G_PRIOR = 0.0629 +K_ITEM = 20 # pseudo-impressions for item CTR shrinkage +K_USER = 5 # pseudo-events for user CTR shrinkage +TOPK = 200 # co-click neighbours kept per item +ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord'] +_t0 = time.time() + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/ev_train.parquet') + TR = pl.read_parquet(f'{D}/rows_train.parquet') + EE = pl.read_parquet(f'{D}/ev_eval.parquet') + ER = pl.read_parquet(f'{D}/rows_eval.parquet') + NI = pl.read_parquet(f'{D}/items.parquet').height + NU = pl.read_parquet(f'{D}/users.parquet').height + return TE, TR, EE, ER, NI, NU + +def _hist_long(E): + return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts'))) + +def click_log(C, P_ev, P_rows, T_ev): + """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories.""" + win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts') + hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C) + CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii') + return CL + +def cooc_tables(CL, NI): + """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next.""" + us = CL.select('u', 'ii').unique() + pop = us.group_by('ii').agg(pl.len().alias('pop')) + pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b')) + .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co'))) + co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b') + .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos')) + .select(pl.col('ii_b').alias('j'), 'ii', 'cos') # j = history item, ii = candidate + .sort('cos', descending=True).group_by('j').head(TOPK)) + seq = CL.select('u', 'ii') # already sorted by u, ts + parts = [] + for lag in (1, 2, 3): + parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j') + .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum()) + tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot')) + trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp')) + .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK)) + return co, trn + +def _segmax(n, idx, vals, init=-1.0): + """v8: == np.maximum.at(np.full(n, init), idx, vals), via sort + reduceat.""" + out = np.full(n, init, np.float32) + if len(idx) == 0: return out + o = np.argsort(idx, kind='stable'); si = idx[o]; sv = vals[o].astype(np.float32) + st = np.flatnonzero(np.r_[True, si[1:] != si[:-1]]) + out[si[st]] = np.maximum(out[si[st]], np.maximum.reduceat(sv, st)); return out + +def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000): + """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy).""" + rng = np.random.default_rng(seed) + u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64) + y = P_rows['clicked'].to_numpy().astype(np.float32) + P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32) + bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32) + b = np.float32(np.log(y.mean() / (1 - y.mean()))) + gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6) + n = len(y) + for ep in range(epochs): + perm = rng.permutation(n) + for s in range(0, n, batch): + idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx] + pu, qi = P[ub], Q[ib] + z = b + bu[ub] + bi[ib] + (pu * qi).sum(1) + g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz + m = len(idx) + Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m)) + Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m)) + dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None] + dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None] + dbu = Su @ g; dbi = Si @ g + gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi + P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ) + bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi) + return dict(b=b, bu=bu, bi=bi, P=P, Q=Q) + +def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + g = float(P_rows['clicked'].mean()) + ndays = (C - J20) / DAY + CL = click_log(C, P_ev, P_rows, T_ev) # per-user stats: train data + each target row's OWN history + CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # item-level / global stats: train data only (no target-event histories) + + # ---------------- item ---------------- + def ictr(c, n): + return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM) + it = P_rows.group_by('ii').agg( + pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'), + (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'), + (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6')) + it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'), + ictr('it_clk6', 'it_imp6').alias('it_ctr6')) + it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'), + (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'), + (pl.col('it_imp') / ndays).alias('it_imp_rate')) + ip = CLg.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'), + (pl.col('ts') < J20).sum().alias('it_nclk_warm'), + ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d')) + + # ---------------- user ---------------- + ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'), + pl.col('ts').max().alias('_u_last_ev')) + ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr')) + uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'), + pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'), + (pl.col('ts') >= C - DAY).sum().alias('us_nclk24')) + uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate')) + + # ---------------- user x item ---------------- + uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'), + (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk')) + uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'), + pl.col('ts').max().alias('_ui_last_imp')) + uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk')) + + # ---------------- assemble ---------------- + R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns]) + .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left') + .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left')) + R = R.with_columns( + pl.col('h').list.len().alias('us_hlen'), + (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'), + pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'), + (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'), + (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'), + (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'), + (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'), + (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'), + pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt')) + # position (from end, 1 = most recent) of candidate in own history + hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii')) + .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own'))) + R = R.join(hx, on='r', how='left') + R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY), + pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY)) + R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in')) + + # ---------------- co-click ---------------- + co, trn = cooc_tables(CLg, NI) + last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20')) + T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left') + X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos']) + .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32)) + .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left') + .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0))) + l5 = pl.col('pos') <= 5 + agg = X.group_by('r').agg( + pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'), + pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'), + pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'), + pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'), + pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'), + pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'), + (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20')) + R = R.join(agg, on='r', how='left').sort('r') + + # ---------------- SVD embeddings ---------------- + # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories; + # 'lag1d' = only clicks/impressions with ts < C - 1 day + if sv_mode == 'std': + CLsv, NEGsrc = CLg, P_rows + elif sv_mode == 'notgt': + CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'), + _hist_long(P_ev).filter(pl.col('ts') < C)]).unique() + NEGsrc = P_rows + elif sv_mode == 'lag1d': + CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY) + else: + raise ValueError(sv_mode) + assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C + cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc + # user-side matrix incl. each target user's own history (fold-in onto item factors learned from train-only clicks) + cnt = CL.filter(pl.col('ts') < C).group_by('u', 'ii').agg(pl.len().alias('n')) if sv_mode == 'std' else cnt + Xu = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) @ sparse.diags(idf) + Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64) + ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos']) + # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway + er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32) + er_l5 = ex['pos'].to_numpy() <= 5 + nR = R.height + svcols = {} + Werj = sparse.csr_matrix((er_w, (er_r, er_j)), shape=(nR, NI)) + for k in (16, 32, 64): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = np.asarray(Xu @ (Vt.T / np.sqrt(S))).astype(np.float32) # fold-in = U*sqrt(S) for train users + Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9) + cand = Vn[iic] + svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1) + svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1) + hv = np.asarray(Werj @ Vn, dtype=np.float32) # v8: sparse matmul replaces np.add.at + hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9) + if k == 32: + svcols['sv_k32_hnorm'] = hn + sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1) + mx = _segmax(nR, er_r, sims); mx5 = _segmax(nR, er_r[er_l5], sims[er_l5]) + svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5 + if k == 64: + # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only) + Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + neg = NEGsrc.filter(pl.col('clicked') == 0) + Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI)) + nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9 + nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9 + cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1) + svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn + for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')): + M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9 + svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()]) + + # ---------------- MF ---------------- + if with_mf: + mf = fit_mf(P_rows, NU, NI) + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + dot = (mf['P'][uu] * mf['Q'][ii]).sum(1) + R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot), + pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii])) + + fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] + + [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] + + [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)]) + + # ---------------- event context ---------------- + main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk', + 'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last', + 'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else []) + exprs = [] + for c in main: + x = pl.col(c).cast(pl.Float64) + exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'), + ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'), + x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')] + R = R.with_columns(exprs) + R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r') + log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s') + return R + +GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf'] + +def feature_cols(df, groups): + cols = [] + for c in df.columns: + if c in ID_COLS: continue + gname = c.split('_')[0] + if gname == 'ev': + src = c[3:].split('_')[0] # ev__... + if 'ev' in groups and src in groups: cols.append(c) + elif gname in groups: + cols.append(c) + return cols diff --git a/v12/sol_work/src10/v3fold.py b/v12/sol_work/src10/v3fold.py new file mode 100644 index 0000000000000000000000000000000000000000..a65c7ac661cea81ee94a30e1026848530fbb0777 --- /dev/null +++ b/v12/sol_work/src10/v3fold.py @@ -0,0 +1,18 @@ +"""v3 features (v2work/v3feat_ne.py = v3/features.py with global/item stats from train data only) for all fold specs. +Usage: python v3fold.py NAME... -> v2work/feat_v3/NAME.parquet + f1..f6: first event/user in [Ck, Ck+1d); val25/val26: [C, end of train); eval: Jan27 cutoff + e1..e4: [Ck, Jan25) (eval-like staleness, for cutoff-25 validation); E1..E6: [Ck, Jan27) (eval-like, final fit)""" +import sys, os +sys.path.insert(0, '@@WORK@@'); sys.path.insert(1, '@@WORK@@/v3') +from v3feat_ne import load, build, log +from split import J20, DAY, first_events +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +OUT = '@@WORK@@/feat_v3'; os.makedirs(OUT, exist_ok=True) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), + **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f'd{k}': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}} +for n in sys.argv[1:]: + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER, TE, TR, NI, NU) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W) + R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU) + R.write_parquet(f'{OUT}/{n}.parquet'); log('wrote', n, R.shape) diff --git a/v12/sol_work/src10/validate.py b/v12/sol_work/src10/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..09008b3ba2c2b62b2d43eb6e2f1c3ee931320ff0 --- /dev/null +++ b/v12/sol_work/src10/validate.py @@ -0,0 +1,18 @@ +"""Standalone validator: python validate.py SUB.csv""" +import sys, csv, math, numpy as np, polars as pl +out=sys.argv[1] +ids=pl.read_csv('@@WORK@@/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8})['rowid'].to_list() +raw=open(out,'rb').read(); errs=[] +try: raw.decode('utf-8') +except Exception: errs.append('not utf-8') +with open(out,newline='',encoding='utf-8') as f: + r=csv.reader(f); h=next(r); rows=list(r) +if h!=['rowid','predicted_probability']: errs.append(f'header {h}') +if len(rows)!=529390: errs.append(f'rows {len(rows)}') +sid=[x[0] for x in rows] +if len(set(sid))!=len(sid): errs.append('dup rowids') +if set(sid)!=set(ids): errs.append('rowid set mismatch') +v=np.array([float(x[1]) for x in rows]) +if not (np.isfinite(v).all() and (v>=0).all() and (v<=1).all()): errs.append('bad values') +print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.5g} max={v.max():.4g} distinct={len(np.unique(v))}') +print('VALID' if not errs else 'INVALID '+';'.join(errs)) diff --git a/v12/sol_work/src10/waitseq.sh b/v12/sol_work/src10/waitseq.sh new file mode 100644 index 0000000000000000000000000000000000000000..6c24eb4b15187bbfb56d5f095731a3176438e176 --- /dev/null +++ b/v12/sol_work/src10/waitseq.sh @@ -0,0 +1,3 @@ +# v10: block until the background sequence stage finishes; fail if it failed +while [ ! -f seq10.rc ]; do sleep 1; done +grep -v ' ep' seq10.log; exit $(cat seq10.rc) diff --git a/v12/sol_work/src10/write_sub.py b/v12/sol_work/src10/write_sub.py new file mode 100644 index 0000000000000000000000000000000000000000..8852ed160597cce378b39fcbb9b5ab1bf43914bc --- /dev/null +++ b/v12/sol_work/src10/write_sub.py @@ -0,0 +1,27 @@ +"""Write + validate submission from eval_pred.parquet. Usage: python write_sub.py OUT.csv""" +import sys, csv, math, hashlib, zipfile, os, numpy as np, polars as pl +out=sys.argv[1] +pr=pl.read_parquet(os.environ.get('PRED','@@WORK@@/eval_pred.parquet')) +ev=pl.read_csv('@@WORK@@/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8}) +df=ev.join(pr,on='rowid',how='left',maintain_order='left') +p=np.nan_to_num(df['p'].to_numpy().astype(np.float64),nan=0.0629,posinf=1.0,neginf=0.0).clip(0,1) +p[np.isnan(df['p'].to_numpy().astype(float))]=0.0629 +print('missing preds filled:',int(df['p'].null_count())) +with open(out,'w',encoding='utf-8',newline='') as f: + f.write('rowid,predicted_probability\n'); f.writelines(f'{r},{v:.8g}\n' for r,v in zip(df['rowid'].to_list(),p)) +# ---- validator +ids=ev['rowid'].to_list(); errs=[] +with open(out,newline='',encoding='utf-8') as f: + r=csv.reader(f); h=next(r); rows=list(r) +if h!=['rowid','predicted_probability']: errs.append('header') +if len(rows)!=529390 or len(ids)!=529390: errs.append(f'rows {len(rows)}') +sid=[x[0] for x in rows] +if len(set(sid))!=len(sid) or set(sid)!=set(ids): errs.append('rowid set mismatch') +vals=[float(x[1]) for x in rows] +if not all(math.isfinite(v) and 0<=v<=1 for v in vals): errs.append('bad values') +v=np.array(vals); print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.4g} max={v.max():.4g} distinct={len(np.unique(v))}') +print('VALID' if not errs else 'INVALID '+';'.join(errs)) +zp=out+'.zip' +with zipfile.ZipFile(zp,'w',zipfile.ZIP_DEFLATED) as z: z.write(out,arcname=os.path.basename(out)) +with zipfile.ZipFile(zp) as z: print('zip contents',z.namelist()) +print('sha256',hashlib.sha256(open(out,'rb').read()).hexdigest()) diff --git a/v12/sol_work/src12/avg_sq.py b/v12/sol_work/src12/avg_sq.py new file mode 100644 index 0000000000000000000000000000000000000000..afec71dbb2fc6157929880b9e79930df5e0d61ab --- /dev/null +++ b/v12/sol_work/src12/avg_sq.py @@ -0,0 +1,17 @@ +"""avg_sq.py OUT DIR1 DIR2 ... : average seq-model outputs across seed runs; event-relative columns recomputed from the averaged logit. +Rows are stored event-major (10 consecutive rows per event) in every dir; rowid order is asserted identical.""" +import sys, os, numpy as np, polars as pl +out, dirs = sys.argv[1], sys.argv[2:]; os.makedirs(out, exist_ok=True) +names = sorted(set.intersection(*[set(f[:-8] for f in os.listdir(d) if f.endswith('.parquet')) for d in dirs])) +for n in names: + T = [pl.read_parquet(f'{d}/{n}.parquet') for d in dirs]; rid = T[0]['rowid'].to_numpy() + for t in T[1:]: assert (t['rowid'].to_numpy() == rid).all(), (n, 'order') + cols = [c for c in T[0].columns if c != 'rowid'] + M = {c: np.mean([t[c].to_numpy().astype(np.float64) for t in T], 0) for c in cols} + S = M['sq_logit'].reshape(-1, 10) + M['sq_z'] = ((S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6)).ravel(); M['sq_dm'] = (S - S.mean(1, keepdims=True)).ravel() + M['sq_lse'] = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1).ravel() + M['sq_rank'] = (np.argsort(np.argsort(-S, 1, kind='stable'), 1) + 1).ravel(); M['sq_gap'] = (S - S.max(1, keepdims=True)).ravel() + M['sq_gap2'] = (S - np.sort(S, 1)[:, -2:-1]).ravel() + M['sq_sd'] = np.std([t['sq_logit'].to_numpy() for t in T], 0) # seed disagreement + df = pl.DataFrame({'rowid': rid, **M}).with_columns(pl.exclude('rowid').cast(pl.Float32)); df.write_parquet(f'{out}/{n}.parquet'); print(n, df.shape, flush=True) diff --git a/v12/sol_work/src12/blend12.py b/v12/sol_work/src12/blend12.py new file mode 100644 index 0000000000000000000000000000000000000000..6c1bc5bd0840a4b381dd5665f317f796814d3a83 --- /dev/null +++ b/v12/sol_work/src12/blend12.py @@ -0,0 +1,17 @@ +"""v12 final blend: z = (dwbig@1000 + dw_s@500 + 2 * mean_over_3_seeds(dwmlp@epoch4)) / 4 (logit blend 1:1:2), p = sigmoid(z). +python blend12.py CUT : CUT=27 -> eval_pred.parquet (rowid, p); CUT=25 -> prints/saves the local AUC on val25.""" +import sys, json, numpy as np, polars as pl +from cl import matrix, BASE +cut = int(sys.argv[1]); vn = 'eval' if cut == 27 else f'val{cut}' +_, m, _ = matrix(vn, 1.0, extra=BASE) +big = np.load(f'ens/dwbig_c{cut}_s0.npy')[:, 0].astype(np.float64); sm = np.load(f'ens/dw_s_c{cut}_s0.npy')[:, 0].astype(np.float64) +ml = np.mean([np.load(f'ens/dwmlp_c{cut}_s{s}.npy')[:, 3].astype(np.float64) for s in (0, 1, 2)], 0) +assert len(big) == len(sm) == len(ml) == m.height +z = (big + sm + 2 * ml) / 4; p = 1 / (1 + np.exp(-z)); assert np.isfinite(p).all() and (p >= 0).all() and (p <= 1).all() +print('corr big/small/mlp', round(float(np.corrcoef(big, sm)[0, 1]), 4), round(float(np.corrcoef(big, ml)[0, 1]), 4)) +if cut == 27: + pl.DataFrame({'rowid': m['rowid'], 'p': p}).write_parquet('eval_pred.parquet'); print('wrote eval_pred.parquet', m.height) +else: + from sklearn.metrics import roc_auc_score + y = m['clicked'].to_numpy(); r = {k: round(roc_auc_score(y, v), 5) for k, v in dict(dwbig=big, dw_s=sm, mlp=ml, blend=z).items()} + print('LOCAL AUC cut', cut, r); json.dump(r, open(f'local_auc_c{cut}.json', 'w')) diff --git a/v12/sol_work/src12/build_folds.py b/v12/sol_work/src12/build_folds.py new file mode 100644 index 0000000000000000000000000000000000000000..8b46fcac55147d81f68085ca2f720d384df3e830 --- /dev/null +++ b/v12/sol_work/src12/build_folds.py @@ -0,0 +1,31 @@ +import polars as pl, sys, os +sys.path.insert(0,'@@WORK@@') +from feats import load, first_events, build, log +D='@@WORK@@'; os.makedirs(f'{D}/feat',exist_ok=True) +TE,TR,EE,ER,NI=load(D) +DAY=86400; J20=1737331200 +END=J20+7*DAY; VC=J20+4*DAY +# validation: V = first event/user in [Jan24,Jan27); training folds end before Jan24 +specs=[(VC,END-VC,'V')]+[(J20+k*DAY,VC-(J20+k*DAY),f'g{k}') for k in (1,2,3)] +if len(sys.argv)>1 and sys.argv[1]=='c25': + specs=[(J20+5*DAY,2*DAY,'val25')]+[(J20+k*DAY,DAY,f'h{k}') for k in (1,2,3,4)] +if len(sys.argv)>1 and sys.argv[1]=='c26': + specs=[(J20+6*DAY,DAY,'val26')]+[(J20+k*DAY,DAY,f'h{k}') for k in (5,)]+[('eval',None,'eval')] +if len(sys.argv)>1 and sys.argv[1]=='eft': + specs=[(J20+k*DAY,(J20+5*DAY)-(J20+k*DAY),f'e{k}') for k in (1,2,3,4)]+[(J20+k*DAY,END-(J20+k*DAY),f'E{k}') for k in (1,2,3,4,5,6)] +if len(sys.argv)>1 and sys.argv[1]=='dft': + specs=[(J20+k*DAY,(J20+6*DAY)-(J20+k*DAY),f'd{k}') for k in (1,2,3,4,5)] +if len(sys.argv)>1 and sys.argv[1]=='final': + specs=[(J20+k*DAY,END-(J20+k*DAY),f'F{k}') for k in (1,2,3,4,5,6)]+[('eval',None,'eval')] +if len(sys.argv)>1 and sys.argv[1]=='lite': # v4_lite: only what the final fit needs (h1..h5 + val26 = f1..f6, eval) + specs=[(J20+k*DAY,DAY,f'h{k}') for k in (3,4,5)]+[(J20+6*DAY,DAY,'val26'),('eval',None,'eval')] +for C,W,name in specs: + out=f'{D}/feat/{name}.parquet' + if os.path.exists(out): continue + if C=='eval': + C=J20+7*DAY; T_ev=EE; T_rows=ER + else: + T_ev=first_events(TE,C,W); T_rows=TR.join(T_ev.select('eid'),on='eid') + R=build(C,T_ev,T_rows,TE,TR,NI) + R.drop('h','h_ts').write_parquet(out) + log('wrote',name,R.shape) diff --git a/v12/sol_work/src12/cl.py b/v12/sol_work/src12/cl.py new file mode 100644 index 0000000000000000000000000000000000000000..432f6981c519d7556ed9628e8aeab7d4ca33a985 --- /dev/null +++ b/v12/sol_work/src12/cl.py @@ -0,0 +1,42 @@ +"""Shared loader + feature-matrix cache (step C). Matrices cached as parquet keyed by hash(table names, feature list, sample).""" +import os, sys, hashlib, json, gc, numpy as np, polars as pl +sys.path.insert(0,'@@WORK@@'); sys.path.insert(1,'@@WORK@@/v3') +from feats import DROP +from features import feature_cols, GROUPS +D='@@WORK@@'; P2=f'{D}/feat'; P3=f'{D}/feat_v3'; CACHE=f'{D}/cache_mx'; os.makedirs(CACHE,exist_ok=True) +V2_OF={'f1':'h1','f2':'h2','f3':'h3','f4':'h4','f5':'h5','f6':'val26','val25':'val25','val26':'val26','eval':'eval'} +V2_OF.update({f'{p}{k}':f'{p}{k}' for p in 'eEd' for k in range(1,7)}) +TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour'] +KEEP=['rowid','eid','u','ts','clicked','us_cold','us_nohist','tm_stale'] +def load_tab(n, extra=(), variant='notime'): + a=pl.read_parquet(f'{P3}/{n}.parquet'); b=pl.read_parquet(f'{P2}/{V2_OF[n]}.parquet') + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.read_parquet(f'{D}/{sub}/{n}.parquet'),on='rowid',how='left') + if '+' in variant: + raise ValueError('mods not used in v7') + return a +def featlist(df, variant='notime'): + f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith(('v2_','x_','r_','s_','ev2_','rc_','sh_','hs_','lr','tA_','tB_'))] + if 'notime' in variant: f=[c for c in f if c not in TIME] + if variant.split('+')[0]=='stale': f=[c for c in f if c not in TIME or c=='tm_stale'] + return f +BASE=('feat_lr2','feat_tt','feat_pp') +def matrix(n, frac=1.0, variant='notime', extra=()): + """v12: base = v8-lite matrix (v2/v3/x + feat_lr2/feat_tt/feat_pp, 'notime'), cached as parquet; every other dir in `extra` + (feat_sqC, feat_dw) is left-joined on rowid and its columns appended. returns (X float32, meta DataFrame, feats).""" + base=tuple(e for e in extra if e in BASE); new=[e for e in extra if e not in BASE]; assert base==BASE, extra + key=hashlib.md5(json.dumps([n,variant,list(base),[os.path.getmtime(f) for f in (f'{P3}/{n}.parquet',f'{P2}/{V2_OF[n]}.parquet',f'{D}/feat_x/{n}.parquet')+tuple(f'{D}/{b}/{n}.parquet' for b in base)]]).encode()).hexdigest()[:12] + fp=f'{CACHE}/{n}_{key}.parquet' + if os.path.exists(fp): t=pl.read_parquet(fp); fl=json.load(open(fp+'.json')) + else: + t=load_tab(n,base,variant); fl=featlist(t,variant) + t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl))) + t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP]) + t.write_parquet(fp+'.tmp'); os.replace(fp+'.tmp',fp); json.dump(fl,open(fp+'.json','w')) + for sub in new: + e=pl.read_parquet(f'{D}/{sub}/{n}.parquet'); nc=[c for c in e.columns if c!='rowid' and c not in fl] + t=t.join(e.select(['rowid']+nc),on='rowid',how='left',maintain_order='left'); fl=fl+nc + X=t.select(fl).cast(pl.Float32).to_numpy(); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect() + return X, meta, fl diff --git a/v12/sol_work/src12/dw/common.py b/v12/sol_work/src12/dw/common.py new file mode 100644 index 0000000000000000000000000000000000000000..352a6b047bad593f1f48a7696321382dd3771606 --- /dev/null +++ b/v12/sol_work/src12/dw/common.py @@ -0,0 +1,56 @@ +"""DW signals shared loader (vendored from dw_signals/common.py; paths -> work dir). +Snapshot(C): everything known from train events with ts < C (rows+labels of those events, plus click log reconstructed from their histories). +""" +import numpy as np, polars as pl, time, gc +from scipy.sparse import csr_matrix +from scipy.sparse.linalg import svds +import os +L=os.environ.get('DW_WORK','@@WORK@@'); DAY=86400; J20=1737331200 # L = work dir holding train_events/train_rows/evaluation_* parquet (prep.py output) +def Jd(d): return J20+(d-20)*DAY +T0=time.time() +def log(*a): print(f'[{time.time()-T0:6.0f}s]',*a,flush=True) + +class Data: + def __init__(s): + ev=pl.read_parquet(f'{L}/train_events.parquet').sort('ts').with_row_index('e') + rows=pl.read_parquet(f'{L}/train_rows.parquet').join(ev.select('eid','e'),on='eid').sort(['e','item']) + s.NE=ev.height; assert rows.height==s.NE*10 + s.ETS=ev['ts'].to_numpy(); s.EID=ev['eid'].to_numpy() + uu,s.EU=np.unique(ev['user'].to_numpy(),return_inverse=True); s.EU=s.EU.astype(np.int32); s.NU=len(uu); s.uu=uu + s.Hlen=ev['h_items'].list.len().to_numpy().astype(np.int64) + s.OFF=np.zeros(s.NE+1,np.int64); s.OFF[1:]=np.cumsum(s.Hlen) + hi=ev['h_items'].explode().drop_nulls().to_numpy(); s.HT=ev['h_ts'].explode().drop_nulls().to_numpy().astype(np.int64) + rr=rows['item'].to_numpy() + s.items=np.unique(np.concatenate([np.unique(rr),np.unique(hi)])); s.NI=len(s.items) + s.HI=np.searchsorted(s.items,hi).astype(np.int32); s.RI=np.searchsorted(s.items,rr).astype(np.int32).reshape(s.NE,10) + s.Y=rows['clicked'].to_numpy().astype(np.int8).reshape(s.NE,10) + s.rowid=rows['rowid'].to_numpy().reshape(s.NE,10) + s.eid2e={} # not needed + s.ev=ev.select('e','eid','user','ts'); s.rows_rowid=rows.select('rowid','e','item') + del ev,rows; gc.collect() + def snap(s,X): + """click log (user,item,ts) from events with ts>20; tm=key&((1<<20)-1) + return dict(nX=nX,cu=(ui//s.NI).astype(np.int32),ci=(ui%s.NI).astype(np.int32),ct=tm*60+Jd(1)) + +def click_svd(S,D,k=32): + """item/user taste vectors from dedup click log (binary, 1/sqrt((nu+1)(ni+1)) scaling). returns item vec (V*s) L2-normed, user vec""" + u,i=S['cu'],S['ci']; key=np.unique(u.astype(np.int64)*D.NI+i); uu=(key//D.NI).astype(np.int32); ii=(key%D.NI).astype(np.int32) + nu=np.bincount(uu,minlength=D.NU).astype(np.float32); ni=np.bincount(ii,minlength=D.NI).astype(np.float32) + M=csr_matrix((1/np.sqrt((nu[uu]+1)*(ni[ii]+1)),(uu,ii)),shape=(D.NU,D.NI),dtype=np.float32) + U,sv,Vt=svds(M,k=k,random_state=0) + V=(Vt.T*sv); V/= np.linalg.norm(V,axis=1,keepdims=True)+1e-9 + return V.astype(np.float32),(U*sv).astype(np.float32) +def norm(x): return x/(np.linalg.norm(x,axis=-1,keepdims=True)+1e-9) +def event_view(a): return a.reshape(-1,10) +def within_event(x): + """z-score, demeaned, rank (0..9 ascending) within the 10-row events; x aligned with val rows""" + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + rk=m.argsort(1).argsort(1).astype(np.float32) + return ((m-mu)/sd).ravel(),(m-mu).ravel(),rk.ravel() diff --git a/v12/sol_work/src12/dw/mkdw.py b/v12/sol_work/src12/dw/mkdw.py new file mode 100644 index 0000000000000000000000000000000000000000..82f41e82203fd545dd4f244959b1832b02dc7b1c --- /dev/null +++ b/v12/sol_work/src12/dw/mkdw.py @@ -0,0 +1,45 @@ +"""mkdw.py NAME... : DW reusable feature families (impression_taste, slate_centrality, skip_contrast) for rows of fold/val/eval table NAME. +Fit cutoff X: f{k} -> J20+k*DAY; val23/25/26 -> J20+3/5/6 d; eval -> J27 (end of train). Only train events ts one empty extra user row.""" + ee=pl.read_parquet(f'{W}/evaluation_events.parquet').sort('ts','eid'); er=pl.read_parquet(f'{W}/evaluation_rows.parquet') + assert ee['ts'].min()>=D.ETS.max() + n=ee.height; eid2e={x:i for i,x in enumerate(ee['eid'].to_list())} + er=er.with_columns(pl.col('eid').replace_strict(eid2e).alias('_e')).sort('_e','rowid'); assert (np.bincount(er['_e'].to_numpy())==10).all() + pos=np.searchsorted(D.uu,ee['user'].to_numpy()); pos=np.minimum(pos,D.NU-1); seen=D.uu[pos]==ee['user'].to_numpy() + new_u=D.NU; eu=np.where(seen,pos,new_u).astype(np.int32); print('eval unseen users',int((~seen).sum()),flush=True) + ri=np.searchsorted(D.items,er['item'].to_numpy()); assert (D.items[ri]==er['item'].to_numpy()).all(); ri=ri.astype(np.int32) + hi=ee['h_items'].explode().drop_nulls().to_numpy(); ht=ee['h_ts'].explode().drop_nulls().to_numpy().astype(np.int64); hl=ee['h_items'].list.len().to_numpy() + hp=np.minimum(np.searchsorted(D.items,hi),D.NI-1); ok=D.items[hp]==hi; print('eval hist items unseen (dropped)',int((~ok).sum()),'of',len(hi),flush=True) + hl=hl.astype(np.int64); cs=np.concatenate([[0],np.cumsum(ok.astype(np.int64))]); ce=np.concatenate([[0],np.cumsum(hl)]); hl2=cs[ce[1:]]-cs[ce[:-1]] + D0=D.NE; D.ETS=np.concatenate([D.ETS,ee['ts'].to_numpy()]); D.EU=np.concatenate([D.EU,eu]); D.Hlen=np.concatenate([D.Hlen,hl2.astype(np.int64)]) + base=D.OFF[-1]; D.OFF=np.concatenate([D.OFF[:-1],base+np.concatenate([[0],np.cumsum(hl2)])]).astype(np.int64) + D.HI=np.concatenate([D.HI,hp[ok].astype(np.int32)]); D.HT=np.concatenate([D.HT,ht[ok]]) + D.RI=np.concatenate([D.RI,ri.reshape(n,10)]); D.Y=np.concatenate([D.Y,np.zeros((n,10),np.int8)]); D.NE=D0+n; D.NU=D.NU+1 + assert len(D.OFF)==D.NE+1 + e=np.repeat(np.arange(D0,D0+n),10) + return dict(rowid=er['rowid'].to_numpy(),e=e,item=ri,user=D.EU[e],ts=D.ETS[e]) +if __name__=='__main__': + D=Data(); names=sys.argv[1:] + for nm in names: + t=time.time(); X=CUT[nm] + if nm=='eval': + nt=D.NE; V=extend_eval(D); assert int(np.searchsorted(D.ETS,X))==nt # snapshot = exactly the train events + else: V=rows_train(D,nm) + df=all_recommended(D,X,V); df=df.with_columns(pl.Series('rowid',V['rowid'])).select(['rowid']+[c for c in df.columns]) + df.write_parquet(f'{OUT}/{nm}.parquet'); s=round(time.time()-t); print(nm,df.shape,f'{s}s',flush=True) diff --git a/v12/sol_work/src12/dw/reusable.py b/v12/sol_work/src12/dw/reusable.py new file mode 100644 index 0000000000000000000000000000000000000000..99eaed74a7543e93fc4c617895043147488e53b4 --- /dev/null +++ b/v12/sol_work/src12/dw/reusable.py @@ -0,0 +1,46 @@ +"""Clean entry points for the recommended new signals. + +All functions take + D : common.Data (lite events/rows loader) + X : fit cutoff, unix seconds (only train events with ts < X are used for any fit/aggregate) + rows : dict from D.val(c) / same layout: 'user','item','ts','e' aligned to n_events*10 rows (10 candidates per event) +and return {feature_name: float32 array of len(rows['item'])} (NaN = no information for that row). + +Leak-safety: every statistic (impression matrix, SVDs, click log) is built from events with ts < X only; the only +per-row inputs are the row's own user id, candidate item, ts and the row's OWN pre-ts history (D.HI/HT/OFF). No eval-row +labels, impressions or histories are ever aggregated. Rows' within-slate stats (centrality) use only the row's own slate. + + from reusable import impression_taste, slate_centrality, skip_contrast, all_recommended +""" +import numpy as np, polars as pl +from common import * +import signals as S_ + +def impression_taste(D,X,rows,ks=(8,16,32,64,128)): + """#1 Impression-SVD taste. SVD(k) of the user x item IMPRESSION-count matrix (train events tspl.DataFrame: + F={}; F.update(impression_taste(D,X,rows)); F.update(slate_centrality(D,X,rows)); F.update(skip_contrast(D,X,rows)) + return pl.DataFrame({k:np.asarray(v,np.float32) for k,v in F.items()}) + +if __name__=='__main__': + import sys; c=int(sys.argv[1]); D=Data(); V=D.val(c); df=all_recommended(D,Jd(c),V); print(df.shape); print(df.columns) diff --git a/v12/sol_work/src12/dw/signals.py b/v12/sol_work/src12/dw/signals.py new file mode 100644 index 0000000000000000000000000000000000000000..02b590b61dad4de52037564ca00859f503879bb1 --- /dev/null +++ b/v12/sol_work/src12/dw/signals.py @@ -0,0 +1,168 @@ +"""Candidate signals. Each function: f(D, S, V, X) -> dict name->float32 array aligned to val rows (len n*10), using ONLY snapshot S (train events ts=X-3*DAY)),u,it,y,t + +def hist_flat(D,V): + """own-history items/ts of each val event, flattened, with row->event pointer""" + e=V['e'][::10]; ln=D.Hlen[e]; off=np.zeros(len(e)+1,np.int64); off[1:]=np.cumsum(ln) + idx=np.concatenate([np.arange(D.OFF[x],D.OFF[x+1]) for x in e]) if len(e) else np.array([],int) + return D.HI[idx],D.HT[idx],off,ln + +def sig_impression_svd(D,S,V,X,k=32,Mimp=None): + """A. taste embedding from IMPRESSIONS (who was shown what), not clicks: SVD of user x item impression-count matrix.""" + nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X) + nu=np.asarray(Mi.sum(1)).ravel(); ni=np.asarray(Mi.sum(0)).ravel() + r,c=Mi.nonzero(); w=np.asarray(Mi[r,c]).ravel() + M=csr_matrix((w/np.sqrt((nu[r]+1)*(ni[c]+1)),(r,c)),shape=Mi.shape,dtype=np.float32) + U,sv,Vt=svds(M,k=k,random_state=0); Vi=(Vt.T*sv).astype(np.float32); Us=(U*sv).astype(np.float32) + u=V['user']; it=V['item']; has=(nu[u]>0) + rec=np.einsum('ij,ij->i',U[u]*np.sqrt(sv),Vt.T[it]*np.sqrt(sv)); + cos=np.einsum('ij,ij->i',norm(Us[u]),norm(Vi[it])) + out=dict(impsvd_rec=rec,impsvd_cos=cos); + for k_,v_ in out.items(): v_[~has]=np.nan + z=within_event(cos); out['impsvd_cos_z']=z[0]; out['impsvd_cos_rk']=z[2] + return {k_:v_.astype(np.float32) for k_,v_ in out.items()},dict(Vi=Vi) + +def sig_skip_embedding(D,S,V,X,Vclk): + """B. negative feedback in embedding space: centroid of items the user was SHOWN but did NOT click (all train before X / last 3d) and centroid of all shown items, cos with candidate (click-SVD item vectors), + plus own-history centroid cos (reference) and skip-minus-history contrasts.""" + nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X) + u=V['user']; it=V['item']; Ec=Vclk[it] + def cen(M): + s=M@Vclk; n=np.asarray(M.sum(1)).ravel(); return s, n + out={} + for nm,M in (('skall',Ms),('skrec',Mr),('impall',Mi)): + s,n=cen(M); cs=norm(s)[u]; c=np.einsum('ij,ij->i',cs,Ec).astype(np.float32); c[n[u]==0]=np.nan; out['c_'+nm]=c + hi,ht,off,ln=hist_flat(D,V) + hs=np.add.reduceat(np.vstack([Vclk[hi],np.zeros((1,Vclk.shape[1]),np.float32)]),np.minimum(off[:-1],len(hi)),axis=0) if len(hi) else None + hs=np.where((ln>0)[:,None],hs,0); hc=np.repeat(norm(hs),10,axis=0) + c=np.einsum('ij,ij->i',hc,Ec).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out['c_hist']=c + out['c_hist_minus_sk']=out['c_hist']-out['c_skall']; out['c_hist_minus_skrec']=out['c_hist']-out['c_skrec'] + # skip-centroid of the user's LAST event slate only (non-clicked), what the user just rejected + return out,{} + +def sig_slate_centrality(D,S,V,X,Vemb,tag='clk'): + """C. slate coherence: candidate's mean / max cosine to the other 9 candidates in the same slate (odd-one-out / retrieved-cluster membership).""" + E=Vemb[V['item']].reshape(-1,10,Vemb.shape[1]); G=np.einsum('nik,njk->nij',E,E); idx=np.arange(10); G[:,idx,idx]=np.nan + mean=np.nanmean(G,2); mx=np.nanmax(G,2); top3=np.sort(np.nan_to_num(G,nan=-9),2)[:,:,-3:].mean(2) + out={f'cent_{tag}_mean':mean.ravel(),f'cent_{tag}_max':mx.ravel(),f'cent_{tag}_top3':top3.ravel()} + out[f'cent_{tag}_mean_dm']=(mean-mean.mean(1,keepdims=True)).ravel() + out[f'slate_{tag}_coh']=np.repeat(mean.mean(1),10) + return {k:v.astype(np.float32) for k,v in out.items()},{} + +def sig_exposure_lift(D,S,V,X): + """E. Pearson-residual of user-item exposure vs independence (u_imp * i_imp / N); same for skipped-only impressions.""" + nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X); N=Mi.sum() + u=V['user']; it=V['item']; ui=np.asarray(Mi[u,it]).ravel(); us=np.asarray(Ms[u,it]).ravel() + nu=np.asarray(Mi.sum(1)).ravel()[u]; ni=np.asarray(Mi.sum(0)).ravel()[it] + ex=nu*ni/N; out={'expo_resid':(ui-ex)/np.sqrt(ex+0.05),'expo_ratio':np.log((ui+0.5)/(ex+0.5)),'skip_cnt':us.astype(np.float32),'skip_frac':(us+0.0)/(ui+1)} + return {k:v.astype(np.float32) for k,v in out.items()},{} + +def sig_coimpression(D,S,V,X): + """F. slate co-membership: for candidate c and each own-history item h, how often h and c were shown in the same train slate (before X), as lift over independence; sum/mean/max over h.""" + nX=int(np.searchsorted(D.ETS,X)); R=D.RI[:nX] + B=csr_matrix((np.ones(nX*10,np.float32),(np.repeat(np.arange(nX),10),R.ravel())),shape=(nX,D.NI)) + C=(B.T@B).tocsr(); C.setdiag(0); C.eliminate_zeros(); ni=np.asarray(B.sum(0)).ravel() + keys=None + hi,ht,off,ln=hist_flat(D,V); n=len(V['e'])//10 + evi=np.repeat(np.arange(n),ln) # event of each history entry + # all (hist item, candidate) pairs: for each event, 20x10 + pe=np.repeat(evi,10); ph=np.repeat(hi,10); pc=V['item'].reshape(n,10)[evi].ravel(); prow=(np.repeat(evi,10)*10+np.tile(np.arange(10),len(evi))) + cnt=np.asarray(C[ph,pc]).ravel().astype(np.float32) + Nimp=nX*10.0; ex=ni[ph]*ni[pc]*9.0/Nimp # P(two given impressions share a slate)=9/N_imp + lift=(cnt-ex)/np.sqrt(ex+0.05) + out={} + for nm,val in (('sum',cnt),('lift',lift)): + s=np.bincount(prow,weights=val,minlength=n*10); out['coimp_'+nm+'_sum']=s + out['coimp_'+nm+'_mean']=s/np.maximum(np.repeat(ln,10),1) + mx=np.full(n*10,-1e9,np.float32); np.maximum.at(mx,prow,lift); mx[mx<-1e8]=np.nan; out['coimp_lift_max']=mx + return {k:np.asarray(v,np.float32) for k,v in out.items()},{} + +def sig_user_time(D,S,V,X): + """H+D. user-level time/behaviour: hour-of-day match of click times / event times, last-event outcome, recent event click rate, same-time-bucket event click rate.""" + nX=int(np.searchsorted(D.ETS,X)); u=V['user']; t=V['ts']; n=len(u)//10 + hr=lambda ts: ((ts%DAY)//3600).astype(np.int64) + # click hours per user from click log + Hc=np.zeros((D.NU,24),np.float32); np.add.at(Hc,(S['cu'],hr(S['ct'])),1) + He=np.zeros((D.NU,24),np.float32); np.add.at(He,(D.EU[:nX],hr(D.ETS[:nX])),1) + h=hr(t) + def win(H): + w=sum(H[u,(h+d)%24] for d in (-2,-1,0,1,2)); tot=H[u].sum(1); return ((w+1)/(tot+24/5.0)/(5/24)).astype(np.float32),tot + out={}; out['hr_click_match'],tc=win(Hc); out['hr_ev_match'],te=win(He) + # last-event outcome / recent click rate / same-bucket ctr (event-level) + evclk=D.Y[:nX].sum(1); eu=D.EU[:nX]; ets=D.ETS[:nX]; tb=((ets%DAY)//21600) + order=np.lexsort((ets,eu)); eus=eu[order]; + last_e=np.full(D.NU,-1,np.int64) + # vectorised: last event index per user = max index among that user's events + np.maximum.at(last_e,eu,np.arange(nX)) + lc=np.where(last_e[u]>=0,evclk[np.maximum(last_e[u],0)],np.nan); out['last_ev_clicked']=lc.astype(np.float32) + # recent-5 event click rate + rank_in_user=np.empty(nX,np.int64); rank_in_user[order]=np.arange(nX)-np.searchsorted(eus,eus) + cnt=np.bincount(eu,minlength=D.NU); frm_end=cnt[eu]-1-rank_in_user # 0 = last + m5=frm_end<5; s5=np.bincount(eu[m5],weights=evclk[m5],minlength=D.NU); n5=np.bincount(eu[m5],minlength=D.NU) + out['ev_ctr_last5']=((s5[u]+0.629*2)/(n5[u]+2)).astype(np.float32) + Sb=np.zeros((D.NU,4),np.float32); Nb=np.zeros((D.NU,4),np.float32); np.add.at(Sb,(eu,tb),evclk); np.add.at(Nb,(eu,tb),1) + tbv=(h//6); out['ev_ctr_samebucket']=((Sb[u,tbv]+0.629*3)/(Nb[u,tbv]+3)).astype(np.float32) + out['ev_cnt_samebucket_frac']=((Nb[u,tbv]+0.5)/(Nb[u].sum(1)+2)).astype(np.float32) + return out,{} + +def _svd_scaled(M,k): + nu=np.asarray(M.sum(1)).ravel(); ni=np.asarray(M.sum(0)).ravel(); M=M.tocoo() + Ms=csr_matrix((M.data/np.sqrt((nu[M.row]+1)*(ni[M.col]+1)),(M.row,M.col)),shape=M.shape,dtype=np.float32) + U,sv,Vt=svds(Ms,k=k,random_state=0); return (U*sv).astype(np.float32),(Vt.T*sv).astype(np.float32),nu + +def sig_impression_variants(D,S,V,X): + """A2. variants of the impression-taste embedding: other ranks, skip-only SVD, joint click+impression SVD, recent-impression centroid, and the user's click-history centroid measured in IMPRESSION item space.""" + nX,Mi,Ms,Mc,Mr,*_=_imp_mats(D,X); u=V['user']; it=V['item']; out={} + def ucos(U_,V_,nu,tag): + c=np.einsum('ij,ij->i',norm(U_[u]),norm(V_[it])).astype(np.float32); c[nu[u]==0]=np.nan; out[tag]=c; out[tag+'_z']=within_event(c)[0] + for k in (16,64): + U_,V_,nu=_svd_scaled(Mi,k); ucos(U_,V_,nu,f'impsvd{k}_cos') + U_,V_,nu=_svd_scaled(Ms,32); ucos(U_,V_,nu,'skipsvd32_cos') + # joint click(log, dedup)+impression + ck=csr_matrix((np.ones(len(S['cu']),np.float32),(S['cu'],S['ci'])),shape=(D.NU,D.NI)); ck.data[:]=1 + from scipy.sparse import hstack + U_,V_,nu=_svd_scaled(hstack([ck*3.0,Mi]).tocsr(),32); Vj=V_[:D.NI] # click-part item vecs, joint user space + c=np.einsum('ij,ij->i',norm(U_[u]),norm(Vj[it])).astype(np.float32); out['joint32_cos_clkpart']=c; out['joint32_cos_clkpart_z']=within_event(c)[0] + Vimp=_svd_scaled(Mi,32)[1]; Vn=norm(Vimp) + # own-history centroid in impression item space + hi,ht,off,ln=hist_flat(D,V); n=len(ln) + hs=np.add.reduceat(np.vstack([Vn[hi],np.zeros((1,32),np.float32)]),np.minimum(off[:-1],len(hi)),axis=0); hs=np.where((ln>0)[:,None],hs,0) + c=np.einsum('ij,ij->i',np.repeat(norm(hs),10,axis=0),Vn[it]).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out['hist_cos_impspace']=c; out['hist_cos_impspace_z']=within_event(c)[0] + # recent (3d) impression centroid in click space needs Vclk -> in compute.py + return out,{} + +def sig_recent_impression(D,S,V,X,Vclk,days=(1,3)): + nX=int(np.searchsorted(D.ETS,X)); u=V['user']; it=V['item']; out={} + for d in days: + m=D.ETS[:nX]>=X-d*DAY; r=np.nonzero(m)[0] + M=csr_matrix((np.ones(len(r)*10,np.float32),(np.repeat(D.EU[r],10),D.RI[r].ravel())),shape=(D.NU,D.NI)); n=np.asarray(M.sum(1)).ravel() + c=np.einsum('ij,ij->i',norm(M@Vclk)[u],Vclk[it]).astype(np.float32); c[n[u]==0]=np.nan; out[f'c_imp{d}d']=c + # decayed by event: also impression count in window of this exact item + out[f'ui_imp{d}d']=np.asarray(M[u,it]).ravel().astype(np.float32) + return out,{} + +def sig_impression_multirank(D,S,V,X,ks=(8,64,128)): + """A3. impression-SVD at more ranks: user-cos, plus own-click-history centroid cos and max-sim (over history items) in that impression item space.""" + nX,Mi,*_=_imp_mats(D,X); u=V['user']; it=V['item']; out={} + hi,ht,off,ln=hist_flat(D,V); n=len(ln); evi=np.repeat(np.arange(n),ln); itm=it.reshape(n,10) + for k in ks: + U_,V_,nu=_svd_scaled(Mi,k); Vn=norm(V_) + if k in (8,128): c=np.einsum('ij,ij->i',norm(U_[u]),Vn[it]).astype(np.float32); c[nu[u]==0]=np.nan; out[f'impsvd{k}_cos']=c + hs=np.zeros((n,k),np.float32); np.add.at(hs,evi,Vn[hi]); c=np.einsum('ij,ij->i',np.repeat(norm(hs),10,axis=0),Vn[it]).astype(np.float32); c[np.repeat(ln==0,10)]=np.nan; out[f'hist_cos_imp{k}']=c + if k==64: # max-sim over history items + mx=np.full((n,10),-1.0,np.float32); H=Vn[hi] + for j in range(10): + s=np.einsum('ik,ik->i',H,Vn[itm[evi,j]]); np.maximum.at(mx[:,j],evi,s) + mx=mx.ravel(); mx[np.repeat(ln==0,10)]=np.nan; out['hist_maxsim_imp64']=mx + for kk in list(out): out[kk+'_z']=within_event(np.nan_to_num(out[kk],nan=np.nanmedian(out[kk])))[0] + return out,{} diff --git a/v12/sol_work/src12/ens.py b/v12/sol_work/src12/ens.py new file mode 100644 index 0000000000000000000000000000000000000000..16416895bca8481413fe41df4a60f8b3f862b356 --- /dev/null +++ b/v12/sol_work/src12/ens.py @@ -0,0 +1,35 @@ +"""LightGBM component of the v12 recipe. python ens.py CUT TAG [--seeds 0] [--drop re] [--params json] [--rounds N] [--save r1,r2] [--lr dirs] [--nfolds 4] [--negfrac f] +CUT=27: train f(CUT-24)..f(CUT-21) (f3..f6), predict eval; CUT=25: train f1..f4, predict val25 (AUC printed). +Negatives of the concatenated training folds are subsampled with one RandomState(123).rand(total) draw. Saves ens/{tag}_c{CUT}_s{seed}.npy (raw scores at the --save rounds).""" +import sys, os, re, json, gc, time, argparse, numpy as np, lightgbm as lgb, polars as pl +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--seeds',default='0') +ap.add_argument('--drop',default=None); ap.add_argument('--params',default='{}'); ap.add_argument('--rounds',type=int,default=400); ap.add_argument('--threads',type=int,default=8) +ap.add_argument('--lr',default=''); ap.add_argument('--nfolds',type=int,default=4); ap.add_argument('--negfrac',type=float,default=1.0); ap.add_argument('--save',default='400') +a=ap.parse_args(); t0=time.time(); os.makedirs('ens',exist_ok=True) +extra=tuple(x for x in a.lr.split(',') if x) +trn=[f'f{k}' for k in range(max(1,a.cut-20-a.nfolds),a.cut-20)] +lens=[len(pl.read_parquet(f'feat_v3/{n}.parquet',columns=['rowid'])) for n in trn] +R=np.random.RandomState(123).rand(sum(lens)) if a.negfrac<1 else None +Xs=[];ys=[];feats=None;off=0 +for n,L in zip(trn,lens): + X,m,f=matrix(n,1.0,extra=extra); feats=f; yy=m['clicked'].to_numpy(); assert len(yy)==L + sel=(yy==1)|(R[off:off+L] v2work/feat_x/NAME.parquet + - con variants: k=32/128, recency-weighted (tau=2d) click/non-click centroids, item-side con (users who clicked i vs skipped i) + - user-cluster item CTR (kmeans on user SVD vectors, 64 & 256 clusters), smoothed toward item CTR + - within-event z-score and rank for each""" +import sys, os, time, numpy as np, polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +from sklearn.cluster import MiniBatchKMeans +sys.path.insert(0, '@@WORK@@/v3') +from features import load, click_log +from split import J20, DAY, first_events +OUT = os.environ.get('X_OUT', '@@WORK@@/feat_x'); os.makedirs(OUT, exist_ok=True) +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'), + **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f'd{k}': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}} +def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9) +def csr(u, i, w): return sparse.csr_matrix((w.astype(np.float32), (u, i)), shape=(NU, NI)) +def build(C, T_ev, T_rows): + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CL = click_log(C, P_ev, P_rows, T_ev) # user-level: train data + each target row's own history + CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # global/item-level: train data only + assert CL['ts'].max() < C and P_rows['ts'].max() < C + R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii') + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + cg = CLg.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = csr(cg['u'].to_numpy(), cg['ii'].to_numpy(), np.ones(cg.height)); Scg = Xc.copy() + cnt = CL.group_by('u', 'ii').agg(pl.len().alias('n'), pl.col('ts').max().alias('lt')) + cu = cnt['u'].to_numpy(); ci = cnt['ii'].to_numpy() + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc + Xu = csr(cu, ci, np.ones(cnt.height)) @ sparse.diags(idf); Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + neg = P_rows.filter(pl.col('clicked') == 0) + nu_ = neg['u'].to_numpy(); ni_ = neg['ii'].to_numpy() + Sc = csr(cu, ci, np.ones(cnt.height)); Sn = csr(nu_, ni_, np.ones(neg.height)) + rc = np.exp(-(C - cnt['lt'].to_numpy()) / (2 * DAY)); rn = np.exp(-(C - neg['ts'].to_numpy()) / (2 * DAY)) + Scr = csr(cu, ci, rc); Snr = csr(nu_, ni_, rn) + F = {} + for k in (32, 64, 128): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + Vn = nrm(Vt.T * np.sqrt(S)); Un = nrm(np.asarray(Xu @ (Vt.T / np.sqrt(S)))); cand = Vn[ii] # user fold-in + nC = nrm(np.asarray(Sc @ Vn)); nN = nrm(np.asarray(Sn @ Vn)) + F[f'x_con05_k{k}'] = (nrm(nC - 0.5 * nN)[uu] * cand).sum(1) + if k == 64: + nCr = nrm(np.asarray(Scr @ Vn)); nNr = nrm(np.asarray(Snr @ Vn)) + for a, nm in ((0.5, '05'), (1.0, '1')): + F[f'x_conr{nm}_k64'] = (nrm(nCr - a * nNr)[uu] * cand).sum(1) + F['x_crcos_k64'] = (nCr[uu] * cand).sum(1) + # item side: centroid of users who clicked item minus users who skipped it, vs target user vector + # item side / clusters: user vectors from TRAIN-ONLY clicks (Ug), so no target (eval) histories enter item-level stats + Ug = nrm(np.asarray(Xc @ (Vt.T / np.sqrt(S)))) + iC = nrm(np.asarray(Scg.T @ Ug)); iN = nrm(np.asarray(Sn.T @ Ug)) + F['x_icon05_k64'] = (nrm(iC - 0.5 * iN)[ii] * Un[uu]).sum(1) + F['x_icos_k64'] = (iC[ii] * Un[uu]).sum(1) + # user-cluster CTR + act = np.unique(np.concatenate([P_rows['u'].to_numpy(), cg['u'].to_numpy()])) + for nc in (64, 256): + km = MiniBatchKMeans(nc, random_state=0, batch_size=4096, n_init=3).fit(Ug[act]) + lab = km.predict(Ug).astype(np.int32) # train users' clusters (train-only vectors) -> cluster CTR tables + labt = km.predict(Un[uu]).astype(np.int32) # target row's cluster from its own user vector (own history allowed) + g = float(P_rows['clicked'].mean()) + pr = P_rows.select('ii', 'clicked', pl.Series('cl', lab[P_rows['u'].to_numpy()])) + ict = pr.group_by('ii').agg(((pl.col('clicked').sum() + 20 * g) / (pl.len() + 20)).alias('ictr')) + cct = pr.group_by('cl', 'ii').agg(pl.col('clicked').sum().alias('c'), pl.len().alias('n')) + q = pl.DataFrame({'cl': labt, 'ii': ii.astype(np.int32), 'r': np.arange(len(uu))}).join(cct, on=['cl', 'ii'], how='left') \ + .join(ict, on='ii', how='left').sort('r').with_columns(pl.col('ictr').fill_null(g), pl.col('c').fill_null(0), pl.col('n').fill_null(0)) + F[f'x_clctr{nc}'] = ((q['c'] + 10 * q['ictr']) / (q['n'] + 10)).to_numpy() + F[f'x_clctr{nc}_lift'] = F[f'x_clctr{nc}'] / q['ictr'].to_numpy() + F[f'x_climp{nc}'] = q['n'].to_numpy().astype(np.float32) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()]) + zc = [c for c in F if not c.startswith('x_climp')] + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in zc] + + [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in zc]) + return R.drop('eid', 'u', 'ii') +for name in sys.argv[1:]: + t0 = time.time(); C, W = SPEC[name] + if W == 'eval': R = build(C, EE, ER) + else: + T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/feats.py b/v12/sol_work/src12/feats.py new file mode 100644 index 0000000000000000000000000000000000000000..652a49954e219d12f1077ea5f88c70571091285e --- /dev/null +++ b/v12/sol_work/src12/feats.py @@ -0,0 +1,224 @@ +"""Feature builder for TechArena CTR. All stats for a cutoff C use only events with ts < C +(plus the target events' own histories, which by construction contain only clicks < their ts).""" +import polars as pl, numpy as np, time +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +DAY = 86400 +K_SMOOTH = 20 + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/train_events.parquet') + TR = pl.read_parquet(f'{D}/train_rows.parquet') + EE = pl.read_parquet(f'{D}/evaluation_events.parquet') + ER = pl.read_parquet(f'{D}/evaluation_rows.parquet') + # global item index + items = pl.concat([TR['item'], ER['item'], TE['h_items'].explode().drop_nulls(), EE['h_items'].explode().drop_nulls()]).unique().sort() + imap = pl.DataFrame({'item': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)}) + def mapi(df, col='item'): + return df.join(imap.rename({'item': col, 'ii': col + '_ii'}), on=col, how='left') + TR = mapi(TR).drop('item').rename({'item_ii': 'ii'}) + ER = mapi(ER).drop('item').rename({'item_ii': 'ii'}) + def map_hist(E): + ex = E.select('eid', 'h_items').explode('h_items').with_row_index('r') + ex = ex.join(imap.rename({'item': 'h_items'}), on='h_items', how='left').sort('r') + h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').drop_nulls().alias('h')) + return E.drop('h_items').join(h, on='eid', how='left').with_columns(pl.col('h').fill_null([])) + TE = map_hist(TE); EE = map_hist(EE) + return TE, TR, EE, ER, len(items) + +def first_events(TE, C, W): + """first event per user in [C, C+W)""" + return (TE.filter((pl.col('ts') >= C) & (pl.col('ts') < C + W)).sort('ts') + .group_by('user', maintain_order=True).first()) + +def build(C, T_ev, T_rows, TE, TR, NI, svd_k=64, with_label=True): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'user', 'ts', 'tb'), on='eid', how='inner') + g = P_rows['clicked'].mean() if len(P_rows) else 0.063 + ndays = max((C - P_ev['ts'].min()) / DAY, 1e-3) if len(P_ev) else 1e-3 + + # ---------- click pool (user, item, ts) ---------- + def hist_clicks(E): + return (E.select('user', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls() + .rename({'h': 'ii', 'h_ts': 'ts'})) + pool = pl.concat([hist_clicks(P_ev), hist_clicks(T_ev), + P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique() + pool = pool.with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64)) + # global/item-level pool: train data only (no target-event histories, so nothing aggregates across eval rows); + # `pool` (with each target row's own history) is used only for that user's own features + poolg = pl.concat([hist_clicks(P_ev), P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique() \ + .with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64)) + log('pool', len(pool)) + + # ---------- item stats ---------- + it = P_rows.group_by('ii').agg(pl.len().alias('i_imp'), pl.col('clicked').sum().alias('i_clk'), + (pl.col('ts') >= C - DAY).sum().alias('i_imp1d'), + pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().alias('i_clk1d')) + it = it.with_columns(((pl.col('i_clk') + K_SMOOTH * g) / (pl.col('i_imp') + K_SMOOTH)).alias('i_ctr'), + ((pl.col('i_clk1d') + K_SMOOTH * g) / (pl.col('i_imp1d') + K_SMOOTH)).alias('i_ctr1d'), + (pl.col('i_imp') / ndays).alias('i_imp_rate')) + ip = poolg.group_by('ii').agg(pl.len().alias('ip_n'), pl.col('user').n_unique().alias('ip_users'), + (pl.col('ts') >= C - DAY).sum().alias('ip_1d'), + (pl.col('ts') >= C - 3 * DAY).sum().alias('ip_3d'), + ((pl.col('ts') - C) / DAY).exp().sum().alias('ip_dec1'), + ((pl.col('ts') - C) / (3 * DAY)).exp().sum().alias('ip_dec3'), + (pl.col('ts') < 1737331200).sum().alias('ip_warm')) + itb = P_rows.group_by('ii', 'tb').agg(pl.len().alias('n'), pl.col('clicked').sum().alias('c')) + itb = itb.with_columns(((pl.col('c') + K_SMOOTH * g) / (pl.col('n') + K_SMOOTH)).alias('i_ctr_tb')).select('ii', 'tb', 'i_ctr_tb') + + # ---------- user stats ---------- + ue = P_rows.group_by('user').agg(pl.col('eid').n_unique().alias('u_ev'), pl.col('clicked').sum().alias('u_clk'), + pl.col('ts').max().alias('u_last_ev')) + ue = ue.with_columns(((pl.col('u_clk') + 5 * g * 10) / (pl.col('u_ev') + 5)).alias('u_ctr')) + up = pool.group_by('user').agg(pl.len().alias('up_n'), pl.col('ts').max().alias('up_last'), + pl.col('ii').n_unique().alias('up_uniq'), + (pl.col('ts') >= C - DAY).sum().alias('up_1d')) + # user-item prior impressions + ui = P_rows.group_by('user', 'ii').agg(pl.len().alias('ui_imp'), pl.col('clicked').sum().alias('ui_clk'), + pl.col('ts').max().alias('ui_last_imp')) + uip = pool.group_by('user', 'ii').agg(pl.len().alias('uip_n'), pl.col('ts').max().alias('uip_last')) + + # ---------- co-occurrence (user sets) & transitions ---------- + us = poolg.select('user', 'ii').unique() + # v8: user-set co-occurrence counts via sparse B^T B (same integer counts as the former self-join, far less memory) + _ur = us['user'].rank('dense').to_numpy().astype(np.int64) - 1 + _B = sparse.csr_matrix((np.ones(len(us), np.int32), (_ur, us['ii'].to_numpy().astype(np.int64))), shape=(int(_ur.max()) + 1 if len(us) else 0, NI)) + _C = (_B.T @ _B).tocsr(); _pop = _C.diagonal().astype(np.float64); _C.setdiag(0); _C.eliminate_zeros() + del _B, _ur + log('cooc pairs', _C.nnz) # _C / _pop replace the (ii, ii_b, co, cos) frame + hash join + seq = poolg.sort('user', 'ts') + tr_pairs = [] + for lag in (1, 2, 3): + tr_pairs.append(seq.with_columns(pl.col('ii').shift(lag).over('user').alias('prev')) + .drop_nulls('prev').select(pl.col('prev').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(tr_pairs).group_by('prev', 'ii').agg(pl.col('w').sum().alias('trn')) + trn_out = trn.group_by('prev').agg(pl.col('trn').sum().alias('trn_tot')) + trn = trn.join(trn_out, on='prev').with_columns((pl.col('trn') / (pl.col('trn_tot') + 5)).cast(pl.Float32).alias('trp')).select('prev', 'ii', 'trn', 'trp') + + # ---------- SVD embeddings of user x item click matrix ---------- + uidx = us.select('user').unique().with_row_index('ur') + m = us.join(uidx, on='user') + df_ = np.bincount(m['ii'].to_numpy(), minlength=NI) + idf = np.log((len(uidx) + 1) / (df_ + 1)).astype(np.float32) + X = sparse.csr_matrix((idf[m['ii'].to_numpy()], (m['ur'].to_numpy(), m['ii'].to_numpy())), shape=(len(uidx), NI)) + rn = np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel()) + 1e-6 + X = sparse.diags(1 / rn) @ X + U, S, Vt = randomized_svd(X, svd_k, random_state=0, n_iter=4) + V = Vt.T.astype(np.float32) # unscaled singular vectors (best in emb_exp) + V /= (np.linalg.norm(V, axis=1, keepdims=True) + 1e-6) + V32 = V[:, :32] / (np.linalg.norm(V[:, :32], axis=1, keepdims=True) + 1e-6) + # fold-in of every target user's own pool (train clicks + own history) onto the train-only factors + uidx = pool.select('user').unique().with_row_index('ur') + mu = pool.select('user', 'ii').unique().join(uidx, on='user') + Xu = sparse.csr_matrix((idf[mu['ii'].to_numpy()], (mu['ur'].to_numpy(), mu['ii'].to_numpy())), shape=(len(uidx), NI)) + Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + U = np.asarray(Xu @ (Vt.T / S)).astype(np.float32) + S = np.ones_like(S) + log('svd done') + + # ---------- assemble target rows ---------- + R = T_rows.join(T_ev, on='eid', how='left').with_row_index('r') + R = R.with_columns((pl.col('ts') - C).alias('stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('hour')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(itb, on=['ii', 'tb'], how='left') + .join(ue, on='user', how='left').join(up, on='user', how='left') + .join(ui, on=['user', 'ii'], how='left').join(uip, on=['user', 'ii'], how='left').sort('r')) + R = R.with_columns( + pl.col('h').list.len().alias('h_len'), + (pl.col('ts') - pl.col('h_ts').list.last()).alias('h_since_last'), + (pl.col('ts') - pl.col('h_ts').list.first()).alias('h_since_first'), + pl.col('h').list.n_unique().alias('h_uniq'), + pl.col('h_ts').list.eval((pl.element() >= pl.element().max() - DAY).sum()).list.first().alias('h_1d_before_last'), + (pl.col('ts') - pl.col('u_last_ev')).alias('u_since_ev'), + (pl.col('ts') - pl.col('up_last')).alias('up_since'), + (pl.col('ts') - pl.col('ui_last_imp')).alias('ui_since_imp'), + (pl.col('ts') - pl.col('uip_last')).alias('uip_since'), + (pl.col('ui_imp') - pl.col('ui_clk')).alias('ui_noclk'), + (pl.col('ip_n') / (pl.col('i_imp') + 1)).alias('ip_per_imp'), + (pl.col('i_ctr_tb') - pl.col('i_ctr')).alias('i_ctr_tb_diff'), + ) + R = R.with_columns((pl.col('h_since_first') / (pl.col('h_len') - 1).clip(1)).alias('h_gap_mean'), + (pl.col('h_uniq') / pl.col('h_len').clip(1)).alias('h_uniq_ratio')) + + # pairwise candidate x history + PR = (R.select('r', 'ii', 'h', 'h_ts', 'ts').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'h_ts', 'pos']).drop_nulls('h') + .with_columns(pl.col('h').cast(pl.Int32))) + _pi = PR['ii'].to_numpy().astype(np.int64); _ph = PR['h'].to_numpy().astype(np.int64) + _cv = np.asarray(_C[_pi, _ph]).ravel().astype(np.float64) + _cs = np.where(_cv > 0, _cv / np.sqrt(np.maximum(_pop[_pi] * _pop[_ph], 1e-12)), 0.0).astype(np.float32) + PR = PR.with_columns(pl.Series('co', _cv.astype(np.int32)), pl.Series('cos', _cs)); del _pi, _ph, _cv, _cs + PR = PR.with_columns((0.85 ** (pl.col('pos') - 1)).alias('rw'), (pl.col('h') == pl.col('ii')).alias('same')) + agg = PR.group_by('r').agg( + pl.col('cos').sum().alias('co_sum'), pl.col('cos').max().alias('co_max'), pl.col('cos').mean().alias('co_mean'), + (pl.col('cos') * pl.col('rw')).sum().alias('co_rw'), pl.col('co').sum().alias('co_cnt'), + pl.col('cos').filter(pl.col('pos') <= 3).sum().alias('co_last3'), + pl.col('same').sum().alias('inh_cnt'), pl.col('pos').filter(pl.col('same')).min().alias('inh_pos'), + (pl.col('ts') - pl.col('h_ts')).filter(pl.col('same')).min().alias('inh_since'), + (pl.col('cos') > 0).sum().alias('co_nz')) + R = R.join(agg, on='r', how='left') + # transitions from last 3 history items + TRp = (R.select('r', 'ii', pl.col('h').list.tail(3).alias('h3')).with_columns(pl.int_ranges(pl.col('h3').list.len(), 0, -1).alias('pos')) + .explode(['h3', 'pos']).drop_nulls('h3').rename({'h3': 'prev'}).with_columns(pl.col('prev').cast(pl.Int32)) + .join(trn, on=['prev', 'ii'], how='left').with_columns(pl.col('trn').fill_null(0), pl.col('trp').fill_null(0))) + tagg = TRp.group_by('r').agg(pl.col('trn').filter(pl.col('pos') == 1).sum().alias('tr_last'), + pl.col('trp').filter(pl.col('pos') == 1).sum().alias('trp_last'), + pl.col('trn').sum().alias('tr_last3'), pl.col('trp').sum().alias('trp_last3')) + R = R.join(tagg, on='r', how='left').sort('r') + # SVD cosine: recency weighted mean of history vectors + ex = PR.select('r', 'h', 'rw') + Wrh = sparse.csr_matrix((ex['rw'].to_numpy().astype(np.float32), (ex['r'].to_numpy(), ex['h'].to_numpy())), shape=(len(R), NI)) # v8: replaces np.add.at + hv = np.asarray(Wrh @ V, dtype=np.float32) + hvn = hv / (np.linalg.norm(hv, axis=1, keepdims=True) + 1e-6) + cand = V[R['ii'].to_numpy()] + svd_cos = (hvn * cand).sum(1) + # user factor from SVD (pool-based, covers clicks beyond last 20) + uf = np.zeros((len(R), V.shape[1]), np.float32) + um = R.select('user').join(uidx, on='user', how='left')['ur'].to_numpy() + ok = ~np.isnan(um.astype(float)) if um.dtype.kind == 'f' else np.ones(len(um), bool) + umi = np.nan_to_num(um.astype(float), nan=0).astype(np.int64) + ufv = (U[umi] * np.sqrt(S)).astype(np.float32); ufv[~ok] = 0 + ufv /= (np.linalg.norm(ufv, axis=1, keepdims=True) + 1e-6) + svd_ucos = (ufv * cand).sum(1) + hv32 = np.asarray(Wrh @ V32, dtype=np.float32) + svd32_cos = (hv32 / (np.linalg.norm(hv32, axis=1, keepdims=True) + 1e-6) * V32[R['ii'].to_numpy()]).sum(1) + extra_svd = {} + for kk in (16, 24, 48): + Vk = V[:, :kk] / (np.linalg.norm(V[:, :kk], axis=1, keepdims=True) + 1e-6) + hk = np.asarray(Wrh @ Vk, dtype=np.float32) + extra_svd[f'svd{kk}_cos'] = (hk / (np.linalg.norm(hk, axis=1, keepdims=True) + 1e-6) * Vk[R['ii'].to_numpy()]).sum(1) + # pairwise svd similarity: max over history, and with the most recent item + pr_r = PR['r'].to_numpy(); pr_c = PR['ii'].to_numpy(); pr_h = PR['h'].to_numpy() + sims = np.empty(len(PR), np.float32) + for k in range(0, len(PR), 2_000_000): + sl = slice(k, k + 2_000_000); sims[sl] = (V[pr_c[sl]] * V[pr_h[sl]]).sum(1) + sp = pl.DataFrame({'r': pr_r, 'sim': sims, 'pos': PR['pos'].to_numpy(), 'same': PR['same'].to_numpy()}).filter(~pl.col('same')) + sagg = sp.group_by('r').agg(pl.col('sim').max().alias('svd_maxsim'), pl.col('sim').filter(pl.col('pos') == 1).max().alias('svd_last'), + pl.col('sim').top_k(3).mean().alias('svd_top3')) + R = R.with_columns(pl.Series('svd_cos', svd_cos), pl.Series('svd_ucos', svd_ucos), pl.Series('svd_hnorm', np.linalg.norm(hv, axis=1)), + pl.Series('svd32_cos', svd32_cos), *[pl.Series(k, v) for k, v in extra_svd.items()]) + R = R.join(sagg, on='r', how='left').sort('r') + + fill0 = ['i_imp', 'i_clk', 'i_imp1d', 'i_clk1d', 'ip_n', 'ip_users', 'ip_1d', 'ip_3d', 'ip_dec1', 'ip_dec3', 'ip_warm', + 'u_ev', 'u_clk', 'up_n', 'up_uniq', 'up_1d', 'ui_imp', 'ui_clk', 'ui_noclk', 'uip_n', 'co_sum', 'co_max', 'co_mean', 'co_rw', + 'co_cnt', 'co_last3', 'inh_cnt', 'co_nz', 'tr_last', 'trp_last', 'tr_last3', 'trp_last3'] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns(pl.col('i_ctr').fill_null(g), pl.col('i_ctr1d').fill_null(g), pl.col('i_ctr_tb').fill_null(g), pl.col('u_ctr').fill_null(g * 10)) + # within-event relative features + rel = ['i_ctr', 'ip_dec1', 'ip_n', 'co_sum', 'co_rw', 'co_max', 'svd_cos', 'svd_ucos', 'svd32_cos', 'svd16_cos', 'svd24_cos', 'svd48_cos', 'svd_maxsim', 'svd_top3', 'tr_last3', 'uip_n', 'i_ctr_tb', 'ui_noclk'] + R = R.with_columns([pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in rel] + + [(pl.col(c) - pl.col(c).mean().over('eid')).alias(c + '_dm') for c in rel] + + [(pl.col(c) / (pl.col(c).max().over('eid') + 1e-9)).alias(c + '_rmax') for c in ['co_sum', 'svd_cos', 'ip_dec1', 'co_rw']]) + R = R.with_columns(pl.col('co_rw').max().over('eid').alias('ev_co_max'), pl.col('svd_cos').max().over('eid').alias('ev_svd_max'), + pl.col('svd_cos').mean().over('eid').alias('ev_svd_mean'), + (pl.col('inh_cnt') > 0).sum().over('eid').alias('ev_n_inh'), pl.col('i_ctr').mean().over('eid').alias('ev_ictr_mean'), + pl.col('ip_dec1').mean().over('eid').alias('ev_ipdec_mean')) + log(f'build C={C} rows={len(R)} {time.time()-t0:.0f}s') + starter = dict(g=g) + return R + +DROP = ['r', 'rowid', 'eid', 'user', 'ii', 'h', 'h_ts', 'ts', 'clicked', 'u_last_ev', 'up_last', 'ui_last_imp', 'uip_last', 'fold'] diff --git a/v12/sol_work/src12/lrfeat2.py b/v12/sol_work/src12/lrfeat2.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee3d585ef8fbe5cf89701ac362aba3692379ce7 --- /dev/null +++ b/v12/sol_work/src12/lrfeat2.py @@ -0,0 +1,60 @@ +"""F1 (data_dig_v1): history-centroid cosine in an exact low-rank SVD space. Train-only, rebuilt at each fold cutoff C. +Matrix: users x items, binary, rows = each user's last N distinct history clicks (from train events with ts < C), +columns scaled by 1/sqrt(pop+5); exact svds (ARPACK) k=17; item vectors = comps 2..K+1 (popularity comp dropped) * s. +Feature: cos(candidate vector, sum of own-history normalised vectors). Within-event _z and _dmax. Eval: fitted at J27 on train only. +Usage: python lrfeat.py NAME... -> feat_lr/NAME.parquet (rowid, lr{K}_cos/_z/_dmax for K in 10,12,16)""" +import sys, os, time, numpy as np, polars as pl, scipy.sparse as sp +from scipy.sparse.linalg import svds +sys.path.insert(0, '@@WORK@@/v3') +from features import load +from split import J20, DAY, first_events +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +N = int(os.environ.get('LR_N', 40)); OUT = os.environ.get('LR_OUT', '@@WORK@@/feat_lr2'); os.makedirs(OUT, exist_ok=True) +KS = tuple(int(x) for x in os.environ.get('LR_KS', '10,12,16').split(',')) +CON = int(os.environ.get('LR_CON', 0)) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None)} +_V = {} +def vecs(C): + if C in _V: return _V[C] + h = (TE.filter(pl.col('ts') < C).select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('t')).unique().sort('u', 't', 'ii')) + h = h.with_columns(pl.int_range(pl.len()).over('u').alias('p'), pl.len().over('u').alias('L')).filter(pl.col('L') - pl.col('p') <= N).unique(['u', 'ii']) + M = sp.csr_matrix((np.ones(h.height, np.float32), (h['u'].to_numpy(), h['ii'].to_numpy())), shape=(NU, NI)) + pop = np.asarray(M.sum(0)).ravel(); M = M.multiply(1 / np.sqrt(pop + 5)).tocsr() + _, s, vt = svds(M, k=max(max(KS) + 1, 17), random_state=0); o = np.argsort(-s); s, vt = s[o], vt[o] + print(time.strftime('%T'), 'C', C, 'pairs', h.height, 'sv', np.round(s[:15], 2), flush=True) + _V[C] = (s, vt); return _V[C] +def build(C, T_ev, T_rows): + s, vt = vecs(C) + E = T_ev.select('eid', 'h').with_row_index('k'); kk = E.select('eid', 'k') + H = E.select('k', 'h').explode('h').drop_nulls('h'); hk = H['k'].to_numpy(); hi = H['h'].to_numpy().astype(np.int64) + R = T_rows.select('rowid', 'eid', 'ii').join(kk, on='eid', how='left'); rk = R['k'].to_numpy(); ri = R['ii'].to_numpy().astype(np.int64) + F = {} + if CON: + P_ev = TE.filter(pl.col('ts') < C).select('eid', 'u') + ng = TR.filter(pl.col('clicked') == 0).join(P_ev, on='eid', how='inner').select('u', 'ii').unique() + ue = T_ev.select('u').with_row_index('k').join(ng, on='u', how='inner'); nk = ue['k'].to_numpy(); ni = ue['ii'].to_numpy().astype(np.int64) + Shk = sp.csr_matrix((np.ones(len(hk)), (hk, hi)), shape=(E.height, NI)) + if CON: Snk = sp.csr_matrix((np.ones(len(nk)), (nk, ni)), shape=(E.height, NI)) + for K in KS: + V = (vt[1:K + 1].T * s[1:K + 1]); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9) + hc = np.asarray(Shk @ V) # v8: sparse matmul replaces np.add.at + hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1) + x = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan) + F[f'lr{K}_cos'] = x.astype(np.float32) + if CON: + nc = np.asarray(Snk @ V) + hn = hc / (np.linalg.norm(hc, axis=1, keepdims=True) + 1e-9); nn = nc / (np.linalg.norm(nc, axis=1, keepdims=True) + 1e-9) + for al, nm in ((0.5, '05'), (1.0, '1')): + d = (hn - al * nn)[rk]; dn = np.linalg.norm(d, axis=1) + F[f'lr{K}_con{nm}'] = np.where(dn > 0, (V[ri] * d).sum(1) / (dn + 1e-9), np.nan).astype(np.float32) + R = R.select('rowid', 'eid').with_columns([pl.Series(c, v).fill_nan(None) for c, v in F.items()]) + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c.replace('_cos', '_z') if c.endswith('_cos') else c + '_z') for c in F] + + [(pl.col(c) - pl.col(c).max().over('eid')).alias(c.replace('_cos', '_dmax') if c.endswith('_cos') else c + '_dmax') for c in F]) + return R.drop('eid') +for n in sys.argv[1:]: + t0 = time.time() + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{n}.parquet'); print(n, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/mlp12.py b/v12/sol_work/src12/mlp12.py new file mode 100644 index 0000000000000000000000000000000000000000..1c7b82ed8c56cbc43ebc86a6a83eaacf6ee2dd7b --- /dev/null +++ b/v12/sol_work/src12/mlp12.py @@ -0,0 +1,42 @@ +"""mlp12.py CUT TAG [--ep 4] [--seed 0]: MLP on the same LightGBM matrix (train f(C-24)..f(C-21), negfrac .5, v10B drop), rank-gauss inputs +from TRAIN quantiles only + missing flags. Saves ens/{TAG}_c{CUT}_s{seed}.npy (n,EP) logits after each epoch.""" +import os, sys, re, time, argparse, json, gc, numpy as np, torch, torch.nn as nn +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--ep',type=int,default=4); ap.add_argument('--seed',type=int,default=0) +ap.add_argument('--lr',default='feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw'); ap.add_argument('--hid',type=int,default=512); ap.add_argument('--drop',type=float,default=0.2) +a=ap.parse_args(); t0=time.time(); torch.manual_seed(a.seed); np.random.seed(a.seed); torch.set_num_threads(int(os.environ.get('MLP_THREADS',8))) +DR='^sq_(pdot|pcos|mdot|mcos)$'; extra=tuple(a.lr.split(',')) +trn=[f'f{k}' for k in range(max(1,a.cut-24),a.cut-20)] +rs=np.random.RandomState(123); Xs=[];ys=[] +for n in trn: + X,m,f=matrix(n,1.0,extra=extra); y=m['clicked'].to_numpy(); sel=(y==1)|(rs.rand(len(y))<0.5); Xs.append(X[sel]); ys.append(y[sel]); feats=f; del X; gc.collect() +keep=[i for i,c in enumerate(feats) if not re.search(DR,c)]; fl=[feats[i] for i in keep] +X=np.concatenate([x[:,keep] for x in Xs]); del Xs; y=np.concatenate(ys).astype(np.float32); gc.collect() +vn='eval' if a.cut==27 else f'val{a.cut}'; Xv,mv,fv=matrix(vn,1.0,extra=extra); Xv=Xv[:,[fv.index(c) for c in fl]] +# rank-gauss via train quantiles (101 knots); NaN -> 0 + flag +from scipy.special import ndtri +qs=np.linspace(0.001,0.999,101); Q=np.nanquantile(X[::7],qs,axis=0).astype(np.float32) # (101,F) +nanc=np.where(np.isnan(X[::7]).any(0)|np.isnan(Xv[::7]).any(0))[0]; G=ndtri(qs).astype(np.float32) +def tr(Z): + out=np.empty((len(Z),Z.shape[1]+len(nanc)),np.float32) + for j in range(Z.shape[1]): + q=np.nan_to_num(Q[:,j],nan=0.0); v=Z[:,j]; u,ix=np.unique(q,return_index=True) + o=np.interp(v,u,G[ix]) if len(u)>1 else np.zeros_like(v); o[np.isnan(v)]=0; out[:,j]=o + out[:,Z.shape[1]:]=np.isnan(Z[:,nanc]); return out +X=np.nan_to_num(tr(X)); Xv=np.nan_to_num(tr(Xv)); gc.collect(); print('prep',X.shape,round(time.time()-t0),'s',flush=True) +D=X.shape[1] +net=nn.Sequential(nn.Linear(D,a.hid),nn.BatchNorm1d(a.hid),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid,a.hid//2),nn.BatchNorm1d(a.hid//2),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid//2,1)) +opt=torch.optim.AdamW(net.parameters(),1e-3,weight_decay=1e-4); lf=nn.BCEWithLogitsLoss(); Xt=torch.from_numpy(X); Yt=torch.from_numpy(y); Xvt=torch.from_numpy(Xv) +bs=2048; steps=a.ep*((len(y)+bs-1)//bs); sch=torch.optim.lr_scheduler.OneCycleLR(opt,2e-3,total_steps=steps,pct_start=0.1); S=[] +for ep in range(a.ep): + net.train(); perm=torch.randperm(len(y)) + for b in range(0,len(y),bs): + j=perm[b:b+bs]; opt.zero_grad(); l=lf(net(Xt[j]).squeeze(-1),Yt[j]); l.backward(); opt.step(); sch.step() + net.eval() + with torch.no_grad(): p=torch.cat([net(Xvt[b:b+65536]).squeeze(-1) for b in range(0,len(Xv),65536)]).numpy() + S.append(p); msg=f'ep{ep} {time.time()-t0:.0f}s' + if a.cut!=27: msg+=f' auc {roc_auc_score(mv["clicked"].to_numpy(),p):.5f}' + print(msg,flush=True) +os.makedirs('ens',exist_ok=True); np.save(f'ens/{a.tag}_c{a.cut}_s{a.seed}.npy',np.stack(S,1).astype(np.float32)) +open('ens/ens_log.jsonl','a').write(json.dumps(dict(cut=a.cut,tag=a.tag,seed=a.seed,min=round((time.time()-t0)/60,1)))+'\n') diff --git a/v12/sol_work/src12/ppmi.py b/v12/sol_work/src12/ppmi.py new file mode 100644 index 0000000000000000000000000000000000000000..0d2fbf205f870114ac1dcb6006f1bc36ad22a94a --- /dev/null +++ b/v12/sol_work/src12/ppmi.py @@ -0,0 +1,59 @@ +"""v8 (ported from v5 ppmi.py): sequence-window PPMI-SVD item embeddings from CLg only (train clicks + train-event histories, ts WORK/feat_pp/NAME.parquet""" +import sys, os, time, json, numpy as np, polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +sys.path.insert(0, '@@WORK@@/v3') +from features import load, click_log +from split import J20, DAY, first_events +OUT = '@@WORK@@/feat_pp'; os.makedirs(OUT, exist_ok=True) +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val23': (J20 + 3 * DAY, None), 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval')} +def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9) +def csr(u, i, w, shape): return sparse.csr_matrix((np.asarray(w, np.float32), (u, i)), shape=shape) +def ppmi_emb(CLg, ks=(32, 64)): + s = CLg.select('u', pl.col('ii').cast(pl.Int64)).to_numpy() # sorted by u, ts + u, it = s[:, 0], s[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32) + for d in range(1, 6): + m = u[d:] == u[:-d]; a, b = it[:-d][m], it[d:][m]; ok = a != b; a, b = a[ok], b[ok] + A = A + csr(a, b, np.full(len(a), 1.0 / d), (NI, NI)) + csr(b, a, np.full(len(a), 1.0 / d), (NI, NI)) + A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum() + pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0 + M = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI)) + E = {} + for k in ks: + U, S, Vt = randomized_svd(M, k, random_state=0, n_iter=5); E[k] = nrm(U * np.sqrt(S)) + return E, int(len(a)) +def build(C, T_ev, T_rows): + P_ev = TE.filter(pl.col('ts') < C); P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CL = click_log(C, P_ev, P_rows, T_ev); CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) + assert CLg['ts'].max() < C and CL['ts'].max() < C + # acceptance: window counts from CLg only -> target events' histories must not change CLg + assert CLg.height == click_log(C, P_ev, P_rows, T_ev.head(0)).height + E, _ = ppmi_emb(CLg) + R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii') + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + cnt = CL.group_by('u', 'ii').agg(pl.len()); Sc = csr(cnt['u'].to_numpy(), cnt['ii'].to_numpy(), np.ones(cnt.height), (NU, NI)) + neg = P_rows.filter(pl.col('clicked') == 0); Sn = csr(neg['u'].to_numpy(), neg['ii'].to_numpy(), np.ones(neg.height), (NU, NI)) + F = {} + E32, E64 = E[32], E[64] + nC = nrm(np.asarray(Sc @ E32)); nN = nrm(np.asarray(Sn @ E32)); F['x_pp_con05_k32'] = (nrm(nC - 0.5 * nN)[uu] * E32[ii]).sum(1) + hC = nrm(np.asarray(Sc @ E64)); F['x_pp_hcos'] = (hC[uu] * E64[ii]).sum(1) + l3 = CL.sort('u', 'ts').group_by('u', maintain_order=True).tail(3).select('u', pl.col('ii').cast(pl.Int64)) + L = np.full((NU, 3), -1, np.int64); pos = l3.with_columns(pl.int_range(pl.len()).over('u').alias('p')) + L[pos['u'].to_numpy(), pos['p'].to_numpy()] = pos['ii'].to_numpy() + best = np.full(len(uu), -np.inf, np.float32); Lu = L[uu] + for j in range(3): + v = Lu[:, j]; ok = v >= 0; sc = np.full(len(uu), -np.inf, np.float32); sc[ok] = (E64[v[ok]] * E64[ii[ok]]).sum(1) + best = np.maximum(best, sc) + F['x_pp_last3'] = np.where(np.isfinite(best), best, 0.0) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()]) + R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in F] + + [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in F]) + return R +def table(name): + C, W = SPEC[name] + if W == 'eval': return build(C, EE, ER) + T_ev = first_events(TE, C, W); return build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) +for name in sys.argv[1:]: + t0 = time.time(); R = table(name).drop('eid', 'u', 'ii'); R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/prep.py b/v12/sol_work/src12/prep.py new file mode 100644 index 0000000000000000000000000000000000000000..5de04f95f2309e945d7dfc0e9fe2d8598625e0eb --- /dev/null +++ b/v12/sol_work/src12/prep.py @@ -0,0 +1,22 @@ +import polars as pl, time, sys +D='@@WORK@@' +def hexid(e, n=15): # opaque id -> int64 via first 15 hex chars after prefix + return e.str.slice(4, n).str.to_integer(base=16) +def prep(name, has_label): + t=time.time() + lf=pl.scan_csv(f'{D}/{name}.csv', schema_overrides={'history_item_ids':pl.Utf8,'history_timestamps':pl.Utf8}) + cols=[pl.col('rowid'), hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('item_id')).alias('item')] + if has_label: cols.append(pl.col('clicked').cast(pl.Int8)) + lf.select(cols).sink_parquet(f'{D}/{name}_rows.parquet') + print(name,'rows done',time.time()-t,flush=True) + ev=(lf.select('event_id','user_id','timestamp','time_bucket','day_type','history_item_ids','history_timestamps') + .unique(subset='event_id',keep='any') + .select(hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('user_id')).alias('user'), + pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'), + pl.col('day_type').cast(pl.Int8).alias('dt'), + pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).list.eval(pl.element().str.slice(4,15).str.to_integer(base=16)).alias('h_items'), + pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts'))) + ev.collect(engine='streaming').sort('ts').write_parquet(f'{D}/{name}_events.parquet') + print(name,'events done',time.time()-t,flush=True) +prep('evaluation',False) +prep('train',True) diff --git a/v12/sol_work/src12/seq11.py b/v12/sol_work/src12/seq11.py new file mode 100644 index 0000000000000000000000000000000000000000..196eaf0018d66fb147c474931af345a51f6d6f6e --- /dev/null +++ b/v12/sol_work/src12/seq11.py @@ -0,0 +1,148 @@ +"""v12 seq11.py (seq10.py + env knobs SQ_SEED/SQ_H/SQ_HID/SQ_DH/SQ_D/SQ_HL/SQ_TEMB; defaults == seq10). Original v10 doc: v10 seq10.py = copy of shipped v8 seq.py + extra outputs for LightGBM stacking (attention stats, user/candidate dot+cos, within-event rank/gap); +fit() reseeds torch per cutoff so each fit is order-independent. Encoder variant via SQ_ENC (din|gru|rec). +v8 sequence model (shipped config: last 500k train events < C, 3 epochs, batch 256, <=6000 steps, NP 3, no slate) (DIN-style target attention), refit per cutoff C on train events ts logit; BCE on all 10 slate rows. +Target tables only use their own history + train events < C (no aggregates over unlabelled target/eval rows). +Usage: python seq.py NAME... [--frac 0.25] -> feat_sq/NAME.parquet (rowid, sq_logit, sq_z, sq_dm, sq_lse)""" +import sys, os, time, argparse, numpy as np, polars as pl, torch, torch.nn as nn +D = os.environ.get('SQ_WORK', '@@WORK@@') +sys.path.insert(0, f'{D}/v3') +from features import load, click_log +from scipy import sparse +from sklearn.utils.extmath import randomized_svd +from split import J20, DAY, first_events +ap = argparse.ArgumentParser(); ap.add_argument('names', nargs='+'); ap.add_argument('--frac', type=float, default=1.0) +ap.add_argument('--ntr', type=int, default=int(os.environ.get('SQ_NTR', 500000))); ap.add_argument('--ep', type=int, default=int(os.environ.get('SQ_EP', 3))) +ap.add_argument('--d', type=int, default=int(os.environ.get('SQ_D', 32))); ap.add_argument('--out', default=os.environ.get('SQ_OUT', f'{D}/feat_sq11')) +a = ap.parse_args(); os.makedirs(a.out, exist_ok=True) +NT = int(os.environ.get('SQ_THREADS', 4)); torch.set_num_threads(NT); torch.manual_seed(0); np.random.seed(0) +DATA = f'{D}/v3/data' +TE, TR, EE, ER, NI, NU = load(DATA) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val23': (J20 + 3 * DAY, None), 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval')} +ENC = os.environ.get('SQ_ENC', 'din'); HL, NP = int(os.environ.get('SQ_HL', 20)), int(os.environ.get('SQ_NP', 3)); SEED = int(os.environ.get('SQ_SEED', 0)); TEMB = int(os.environ.get('SQ_TEMB', 0)); HID = int(os.environ.get('SQ_HID', 64)); SLATE = int(os.environ.get('SQ_SLATE', 0)) +def slates(EV, RW): + """events EV (eid,u,ts,tb,dt,h,h_ts) + rows -> arrays in EV order; candidate matrix [E,10], clicks, rowids.""" + R = RW.join(EV.select('eid', pl.int_range(pl.len()).alias('_e')), on='eid', how='inner').sort('_e', 'rowid') + e = R['_e'].to_numpy(); assert (np.bincount(e, minlength=EV.height) == 10).all() + cand = R['ii'].to_numpy().reshape(-1, 10).astype(np.int64) + clk = R['clicked'].to_numpy().reshape(-1, 10).astype(np.float32) if 'clicked' in R.columns else None + return cand, clk, R['rowid'].to_numpy().reshape(-1, 10) +# global train index sorted by (u, ts, eid): used for prior impressions +TS = TE.sort('u', 'ts', 'eid'); T_u = TS['u'].to_numpy().astype(np.int64); T_ts = TS['ts'].to_numpy() +T_cand, T_clk, _ = slates(TS, TR); KEY = T_u * (1 << 32) + T_ts +def tokens(EV, C): + """[E, HL+NP*10] item / flag / gap-bucket tokens + ctx ints.""" + n = EV.height; u = EV['u'].to_numpy().astype(np.int64); ts = EV['ts'].to_numpy() + it = np.full((n, HL + NP * 10), NI, np.int64); fl = np.zeros_like(it); gp = np.zeros_like(it) + hh = EV['h'].to_list(); ht = EV['h_ts'].to_list() + for i in range(n): + l = hh[i] + if l: + l = l[-HL:]; t = np.asarray(ht[i][-HL:]); k = len(l) + it[i, :k] = l; fl[i, :k] = 1; gp[i, :k] = 1 + np.clip(np.log2(np.maximum(ts[i] - t, 60) / 60), 0, 19).astype(np.int64) + lim = np.minimum(ts, C); pos = np.searchsorted(KEY, u * (1 << 32) + lim, 'left') - 1 # last train event with ts < lim + npr = np.zeros(n, np.int64) + for j in range(NP): + p = pos - j; ok = (p >= 0) & (T_u[np.maximum(p, 0)] == u); p = np.maximum(p, 0) + assert (T_ts[p][ok] < lim[ok]).all() and (T_ts[p][ok] < C).all() + s = slice(HL + 10 * j, HL + 10 * (j + 1)) + it[:, s] = np.where(ok[:, None], T_cand[p], NI); fl[:, s] = np.where(ok[:, None], 2 + T_clk[p].astype(np.int64), 0) + g = 1 + np.clip(np.log2(np.maximum(ts - T_ts[p], 60) / 60), 0, 19).astype(np.int64); gp[:, s] = np.where(ok, g, 0)[:, None] + npr += ok + ctx = np.stack([EV['tb'].to_numpy().astype(np.int64), EV['dt'].to_numpy().astype(np.int64), (fl[:, :HL] > 0).sum(1), npr], 1) + return it, fl, gp, ctx +def ppmi_E(C, k=64): + P_ev = TE.filter(pl.col('ts') < C); P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + CLg = click_log(C, P_ev, P_rows, P_ev.head(0)); assert CLg['ts'].max() < C + s_ = CLg.select('u', pl.col('ii').cast(pl.Int64)).to_numpy(); u, it = s_[:, 0], s_[:, 1]; A = sparse.csr_matrix((NI, NI), dtype=np.float32) + for d in range(1, 6): + m = u[d:] == u[:-d]; x, y = it[:-d][m], it[d:][m]; ok = x != y; x, y = x[ok], y[ok]; w = np.full(len(x), 1.0 / d, np.float32) + A = A + sparse.csr_matrix((w, (x, y)), shape=(NI, NI)) + sparse.csr_matrix((w, (y, x)), shape=(NI, NI)) + A = A.tocoo(); r = np.asarray(A.sum(1)).ravel(); c = np.asarray(A.sum(0)).ravel(); T = A.data.sum() + pmi = np.log(A.data * T / (r[A.row] * c[A.col])) - np.log(5); k_ = pmi > 0 + Mx = sparse.csr_matrix((pmi[k_].astype(np.float32), (A.row[k_], A.col[k_])), shape=(NI, NI)) + U, S, _ = randomized_svd(Mx, k, random_state=0, n_iter=5); V = U * np.sqrt(S); V = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9) + return np.vstack([V, np.zeros((1, k))]).astype(np.float32) +class M(nn.Module): + def __init__(s, EMB, d=32, H=2, dh=16): + super().__init__(); s.H, s.dh = H, dh + s.E0 = nn.Embedding.from_pretrained(torch.from_numpy(EMB), freeze=True); s.P = nn.Linear(EMB.shape[1], d); s.S = nn.Embedding(2, H) + s.T = nn.Embedding(EMB.shape[0], d) if TEMB else None + if TEMB: nn.init.normal_(s.T.weight, std=0.01) + s.E = (lambda x: s.P(s.E0(x)) + s.T(x)) if TEMB else (lambda x: s.P(s.E0(x))); s.F = nn.Embedding(4, d); s.G = nn.Embedding(21, d); + s.c = nn.ModuleList([nn.Embedding(4, 4), nn.Embedding(2, 2), nn.Embedding(HL + 1, 4), nn.Embedding(NP + 1, 2)]) + s.q = nn.Linear(d, H * dh); s.k = nn.Linear(d, H * dh); s.v = nn.Linear(d, H * dh); s.null = nn.Parameter(torch.zeros(1, 1, H, dh)) + if ENC == 'rec': s.R = nn.Embedding(21, d) + if ENC == 'gru': s.gru = nn.GRU(d, d, batch_first=True) + s.mlp = nn.Sequential(nn.Linear(d * (1 + 2 * SLATE) + H * dh * (1 + SLATE) + 12, HID), nn.ReLU(), nn.Linear(HID, 1)) + def forward(s, it, fl, gp, ctx, cand, extra=False): + B, T = it.shape; K = s.E(it) + s.F(fl) + s.G(gp); q = s.E(cand) # (B,T,d), (B,10,d) + if ENC == 'rec': K = K + s.R(gp) # recency-aware keys (extra gap emb) + if ENC == 'gru': # GRU over tokens in reverse-recency order -> contextualised keys + K = K + s.gru(K)[0] + Q = s.q(q).view(B, 10, s.H, s.dh); Kk = s.k(K).view(B, T, s.H, s.dh); V = s.v(K).view(B, T, s.H, s.dh) + same = (cand[:, :, None] == it[:, None, :]).long(); att = torch.einsum('bchd,bthd->bcht', Q, Kk) / s.dh ** 0.5 + s.S(same).permute(0, 1, 3, 2); att = att.masked_fill((fl == 0)[:, None, None, :], -1e9) + an = (Q * s.null).sum(-1, keepdim=True) / s.dh ** 0.5; w = torch.softmax(torch.cat([att, an], -1), -1) + P = torch.einsum('bcht,bthd->bchd', w[..., :T], V).reshape(B, 10, -1) + cx = torch.cat([e(ctx[:, i]) for i, e in enumerate(s.c)], -1)[:, None].expand(B, 10, 12) + f = [q, P, cx] + if SLATE: # within-event context: the slate's own candidates only + qm = q.mean(1, keepdim=True).expand_as(q); Pm = P.mean(1, keepdim=True).expand_as(P); f = [q, q - qm, qm, P, P - Pm, cx] + out = s.mlp(torch.cat(f, -1)).squeeze(-1) + if not extra: return out + valid = (fl > 0).float(); nv = valid.sum(1).clamp(min=1) # (B,T),(B,) + wm = w.mean(2) # head-mean weights (B,10,T+1) + a_max = wm[..., :T].max(-1).values; a_null = wm[..., T]; a_mean = (1 - a_null) / nv[:, None] + ent = -(wm.clamp(min=1e-9).log() * wm).sum(-1) + vq = s.v(q) # candidate in value space (B,10,H*dh) + pdot = (P * vq).sum(-1); pcos = pdot / (P.norm(dim=-1) * vq.norm(dim=-1) + 1e-6) + mu = (K * valid[..., None]).sum(1) / nv[:, None] # mean-pooled user emb (B,d) + mdot = (q * mu[:, None]).sum(-1); mcos = mdot / (q.norm(dim=-1) * mu.norm(dim=-1)[:, None] + 1e-6) + ck = ((fl == 1) | (fl == 3)).float(); e0 = s.E0(it); eu = (e0 * ck[..., None]).sum(1); ec = s.E0(cand) + ecos = (ec * eu[:, None]).sum(-1) / (ec.norm(dim=-1) * eu.norm(dim=-1)[:, None] + 1e-6) + return out, torch.stack([a_max, a_null, a_mean, ent, pdot, pcos, mdot, mcos, ecos], -1) +XN = ['sq_amax', 'sq_anull', 'sq_amean', 'sq_aent', 'sq_pdot', 'sq_pcos', 'sq_mdot', 'sq_mcos', 'sq_ecos'] +def usel(u): return (u * 2654435761 % 2 ** 32) / 2 ** 32 < a.frac +_M = {} +def fit(C): + if C in _M: return _M[C] + torch.manual_seed(SEED); np.random.seed(SEED) + t0 = time.time(); m = T_ts < C; idx = np.nonzero(m)[0]; idx = idx[np.argsort(T_ts[idx], kind='stable')][-a.ntr:] # train on all users; --frac only subsamples target users + it, fl, gp, ctx = tokens(TS[idx], C); cand = T_cand[idx]; y = T_clk[idx]; tk = time.time() - t0 + X = [torch.from_numpy(z) for z in (it, fl, gp, ctx, cand)]; Y = torch.from_numpy(y) + te = time.time(); EMB = ppmi_E(C); print(f' ppmi C={C} {time.time() - te:.0f}s', flush=True) + mdl = M(EMB, a.d, int(os.environ.get('SQ_H', 2)), int(os.environ.get('SQ_DH', 16))); opt = torch.optim.Adam(mdl.parameters(), float(os.environ.get('SQ_LR', 2e-3))); lossf = nn.BCEWithLogitsLoss(); bs = int(os.environ.get('SQ_BS', 256)); n = len(idx) + maxst = int(os.environ.get('SQ_MAXSTEPS', 6000)); st = 0 # v8: hard cap on optimizer steps -> bounded eval-time fit + for ep in range(a.ep): + perm = torch.randperm(n); tl = 0 + for b in range(0, n, bs): + if st >= maxst: break + st += 1; j = perm[b:b + bs]; opt.zero_grad(); l = lossf(mdl(*[x[j] for x in X]), Y[j]); l.backward(); opt.step(); tl += l.item() * len(j) + print(f' C={C} ep{ep} loss {tl / n:.4f} n={n} steps={st} {time.time() - t0:.0f}s', flush=True) + mdl.eval(); _M[C] = mdl; print(f' fit C={C} tok {tk:.0f}s total {time.time() - t0:.0f}s', flush=True); return mdl +def score(mdl, EV, C, cand): + it, fl, gp, ctx = tokens(EV, C); out = []; ex = [] + with torch.no_grad(): + for b in range(0, len(cand), 8192): + o, e = mdl(*[torch.from_numpy(z[b:b + 8192]) for z in (it, fl, gp, ctx, cand)], extra=True); out.append(o.numpy()); ex.append(e.numpy()) + return np.concatenate(out), np.concatenate(ex) +TIM = {} +for name in a.names: + t0 = time.time(); name, _, mn = name.partition('@'); C, W = SPEC[name]; CM = SPEC[mn][0] if mn else C # 'f4@f3': score f4 rows (history at C_f4) with the model fit at C_f3 < C_f4 + assert CM <= C + if W == 'eval': EV, RW = EE, ER + else: EV = first_events(TE, C, W); RW = TR.join(EV.select('eid'), on='eid', how='semi') + if a.frac < 1: EV = EV.filter(pl.Series(usel(EV['u'].to_numpy().astype(np.int64)))) + cand, _, rid = slates(EV, RW); S, X = score(fit(CM), EV, C, cand) + z = (S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-6); lse = np.log(np.exp(S).sum(1, keepdims=True)).repeat(10, 1) + rk = (np.argsort(np.argsort(-S, 1, kind='stable'), 1) + 1).astype(np.float32) # 1 = event top + gap = S - S.max(1, keepdims=True); gap2 = S - np.sort(S, 1)[:, -2:-1] # gap to top / to runner-up + cols = {'rowid': rid.ravel(), 'sq_logit': S.ravel(), 'sq_z': z.ravel(), 'sq_dm': (S - S.mean(1, keepdims=True)).ravel(), 'sq_lse': lse.ravel(), + 'sq_rank': rk.ravel(), 'sq_gap': gap.ravel(), 'sq_gap2': gap2.ravel()} + for i, n_ in enumerate(XN): cols[n_] = X[..., i].ravel() + for n_ in ('sq_pcos', 'sq_mcos', 'sq_ecos'): # within-event demeaned versions + v = X[..., XN.index(n_)]; cols[n_ + '_dm'] = (v - v.mean(1, keepdims=True)).ravel() + df = pl.DataFrame(cols).with_columns(pl.exclude('rowid').cast(pl.Float32)) + df.write_parquet(f'{a.out}/{name}.parquet'); print(name, df.shape, f'{time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/seqrun.py b/v12/sol_work/src12/seqrun.py new file mode 100644 index 0000000000000000000000000000000000000000..2dea8d9c37d443cc8f075297a68b005c4e6c2f82 --- /dev/null +++ b/v12/sol_work/src12/seqrun.py @@ -0,0 +1,33 @@ +"""seqrun.py NAMES... : multi-seed sequence-model stage. Runs seq11.py once per seed (SQ_SEED = 0..V12_SQ_SEEDS-1) with at most V12_SQ_PAR runs +at the same time, SQ_THREADS torch threads each, then averages the seeds into feat_sqC/ (avg_sq.py). +Seeds 0-7: base config (== seq10); seeds 8-10: SQ_H=4 SQ_HID=128 (4 heads, hidden 128) for ensemble diversity. +Env: V12_SQ_SEEDS (default 11), V12_SQ_PAR (default min(3, threads//2)), V12_SQ_THREADS (torch threads per run, default threads//par).""" +import os, sys, time, subprocess +W = '@@WORK@@'; names = sys.argv[1:] +NS = int(os.environ.get('V12_SQ_SEEDS', 11)); T = int(os.environ.get('V12_THREADS', 8)) +PAR = int(os.environ.get('V12_SQ_PAR', max(1, min(3, T // 2)))); TH = int(os.environ.get('V12_SQ_THREADS', max(1, T // PAR))) +print(f'seqrun: {NS} seeds, {PAR} concurrent, {TH} threads each, tables {names}', flush=True) +assert NS >= 1 +def cfg(s): return {'SQ_H': '4', 'SQ_HID': '128'} if s >= 8 else {} +todo = list(range(NS)); run = {}; t0 = time.time(); done = [] +os.makedirs(f'{W}/msC/logs', exist_ok=True) +def finished(s): return all(os.path.exists(f'{W}/msC/s{s}/{n}.parquet') for n in names) and os.path.exists(f'{W}/msC/s{s}/DONE') +todo = [s for s in todo if not finished(s)]; print('seeds to run', todo, flush=True) +while todo or run: + while todo and len(run) < PAR: + s = todo.pop(0); od = f'{W}/msC/s{s}'; os.makedirs(od, exist_ok=True) + env = dict(os.environ, SQ_SEED=str(s), SQ_THREADS=str(TH), SQ_OUT=od, SQ_WORK=W, SQ_NTR='500000', POLARS_MAX_THREADS=str(TH), OMP_NUM_THREADS=str(TH), OPENBLAS_NUM_THREADS=str(TH), MKL_NUM_THREADS=str(TH), PYTHONUNBUFFERED='1', **cfg(s)) + lf = open(f'{W}/msC/logs/s{s}.log', 'w') + run[s] = (subprocess.Popen([sys.executable, 'seq11.py'] + names, cwd=W, env=env, stdout=lf, stderr=subprocess.STDOUT), time.time()); print(f'[{time.time() - t0:.0f}s] start seed {s}', flush=True) + time.sleep(15) # stagger the memory-heavy start-up + time.sleep(3) + for s, (p, ts) in list(run.items()): + r = p.poll() + if r is None: continue + del run[s] + if r != 0: + for q, _ in run.values(): q.kill() + sys.exit(f'seq seed {s} failed (rc {r}); see {W}/msC/logs/s{s}.log') + open(f'{W}/msC/s{s}/DONE', 'w').write('ok'); print(f'[{time.time() - t0:.0f}s] seed {s} done in {time.time() - ts:.0f}s', flush=True) +subprocess.run([sys.executable, 'avg_sq.py', f'{W}/feat_sqC'] + [f'{W}/msC/s{s}' for s in range(NS)], cwd=W, check=True) +print(f'seqrun done {time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/ttfeat.py b/v12/sol_work/src12/ttfeat.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1e9bdb817c6edf397fb30603cc806edbefa529 --- /dev/null +++ b/v12/sol_work/src12/ttfeat.py @@ -0,0 +1,82 @@ +"""data_dig_v2 features, refit per cutoff C on train only (events ts < C); eval fitted at J27 on all train. +(A) row-scaled exact SVD: binary user x item of ALL pre-C history clicks, diag(1/sqrt(len_u)) M diag(1/sqrt(pop+5)), svds k16 rs0, + comps 2..13 * s -> tA_cos = cos(candidate, own-history centroid) (+ _z, _dmax within event). +(B) two-tower conditional logit, tied item vectors initialised from L2-normalised (A) rows, u = mean of last<=20 own-history vectors, + score u.v, 10-way softmax CE on clicked train events (ts 0 + cand = torch.from_numpy(np.array(G['ii'].to_list(), np.int64)[keep]); lab = torch.from_numpy(clk[keep].argmax(1).astype(np.int64)) + Ht = torch.from_numpy(hmat(G.filter(pl.Series(keep)))) + torch.manual_seed(0); np.random.seed(0) + P = torch.nn.Embedding(NI + 1, 12, padding_idx=NI) + with torch.no_grad(): P.weight[:NI] = torch.from_numpy(nrm(VA)); P.weight[NI] = 0 + opt = torch.optim.Adam(P.parameters(), lr=1e-3); N = len(lab) + def uvec(H): + m = (H < NI).float(); return (P(H) * m[..., None]).sum(1) / m.sum(1, keepdim=True).clamp(min=1) + for ep in range(6): + perm = torch.randperm(N); tot = 0.0 + for i in range(0, N, 2048): + j = perm[i:i + 2048]; sc = (uvec(Ht[j])[:, None, :] * P(cand[j])).sum(-1) + loss = torch.nn.functional.cross_entropy(sc, lab[j]); opt.zero_grad(); loss.backward(); opt.step() + with torch.no_grad(): P.weight[NI] = 0 + tot += loss.item() * len(j) + print(f' C {C} ep{ep} loss {tot / N:.4f}', flush=True) + VB = P.weight.detach().numpy().copy() + print(time.strftime('%T'), 'C', C, 'train events', N, f'{time.time() - t0:.0f}s', flush=True) + _M[C] = (VA, VB); return _M[C] +def build(C, T_ev, T_rows): + VA, VB = fit(C) + E = T_ev.select('eid', 'h').with_row_index('k') + R = T_rows.select('rowid', 'eid', 'ii').join(E.select('eid', 'k'), on='eid', how='left') + rk = R['k'].to_numpy(); ri = R['ii'].to_numpy().astype(np.int64) + # (A) all-history centroid cosine + V = nrm(VA); H = E.select('k', 'h').explode('h').drop_nulls('h'); hc = np.asarray(sp.csr_matrix((np.ones(H.height), (H['k'].to_numpy(), H['h'].to_numpy().astype(np.int64))), shape=(E.height, V.shape[0])) @ V) # v8: replaces np.add.at + hcr = hc[rk]; nr = np.linalg.norm(hcr, axis=1) + a = np.where(nr > 0, (V[ri] * hcr).sum(1) / (nr + 1e-9), np.nan) + # (B) tower dot with u = mean of last <= 20 own-history vectors + Hm = hmat(E); m = (Hm < NI); Vp = np.vstack([VB[:NI], np.zeros((1, VB.shape[1]), VB.dtype)]) + u = (Vp[Hm] * m[..., None]).sum(1) / np.maximum(m.sum(1, keepdims=True), 1) + dot = np.where(m.sum(1)[rk] > 0, (u[rk] * VB[ri]).sum(1), np.nan) + R = R.select('rowid', 'eid').with_columns(pl.Series('tA_cos', a.astype(np.float32)).fill_nan(None), pl.Series('tB_dot', dot.astype(np.float32)).fill_nan(None)) + R = R.with_columns(((pl.col('tA_cos') - pl.col('tA_cos').mean().over('eid')) / (pl.col('tA_cos').std().over('eid') + 1e-6)).alias('tA_z'), + (pl.col('tA_cos') - pl.col('tA_cos').max().over('eid')).alias('tA_dmax'), + pl.col('tB_dot').exp().sum().over('eid').log().alias('tB_lse'), pl.col('tB_dot').mean().over('eid').alias('tB_mean'), + (pl.col('tB_dot') - pl.col('tB_dot').mean().over('eid')).alias('tB_dmean'), + ((pl.col('tB_dot') - pl.col('tB_dot').mean().over('eid')) / (pl.col('tB_dot').std().over('eid') + 1e-6)).alias('tB_dot_z')) + return R.drop('eid') +for n in sys.argv[1:]: + t0 = time.time() + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi')) + R.write_parquet(f'{OUT}/{n}.parquet'); print(n, R.shape, f'{time.time() - t0:.0f}s', flush=True) diff --git a/v12/sol_work/src12/v3/features.py b/v12/sol_work/src12/v3/features.py new file mode 100644 index 0000000000000000000000000000000000000000..c1dac35adfde2286a866d8ea1d6c94a1a5e2fe10 --- /dev/null +++ b/v12/sol_work/src12/v3/features.py @@ -0,0 +1,285 @@ +"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR. + +build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate. +Every statistic uses ONLY: + * train events/rows with ts < C (fit window: impressions + labels), and + * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts). +Since target events are each user's first event at/after C, their histories contain only clicks < C, +so nothing observed after a target row's own event time is ever joined. + +Feature groups (column prefixes, used for ablation): + hist_ candidate vs the user's click log / own history / past impressions of that user-item pair + it_ item popularity & CTR (full fit window, last 24h, last 6h, trend) + co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks + us_ user activity / CTR / recency / cold-start flags + ev_ within-event rank / z-score / event max & mean of the main features + tm_ time_bucket, day_type, hour, staleness since cutoff + sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines + mf_ logistic matrix factorisation score fitted on fit-window impressions +""" +import time +import numpy as np +import polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +J20 = 1737331200 +DAY = 86400 +G_PRIOR = 0.0629 +K_ITEM = 20 # pseudo-impressions for item CTR shrinkage +K_USER = 5 # pseudo-events for user CTR shrinkage +TOPK = 200 # co-click neighbours kept per item +ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord'] +_t0 = time.time() + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/ev_train.parquet') + TR = pl.read_parquet(f'{D}/rows_train.parquet') + EE = pl.read_parquet(f'{D}/ev_eval.parquet') + ER = pl.read_parquet(f'{D}/rows_eval.parquet') + NI = pl.read_parquet(f'{D}/items.parquet').height + NU = pl.read_parquet(f'{D}/users.parquet').height + return TE, TR, EE, ER, NI, NU + +def _hist_long(E): + return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts'))) + +def click_log(C, P_ev, P_rows, T_ev): + """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories.""" + win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts') + hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C) + CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii') + return CL + +def cooc_tables(CL, NI): + """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next.""" + us = CL.select('u', 'ii').unique() + pop = us.group_by('ii').agg(pl.len().alias('pop')) + pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b')) + .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co'))) + co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b') + .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos')) + .select(pl.col('ii_b').alias('j'), 'ii', 'cos') # j = history item, ii = candidate + .sort('cos', descending=True).group_by('j').head(TOPK)) + seq = CL.select('u', 'ii') # already sorted by u, ts + parts = [] + for lag in (1, 2, 3): + parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j') + .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum()) + tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot')) + trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp')) + .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK)) + return co, trn + +def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000): + """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy).""" + rng = np.random.default_rng(seed) + u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64) + y = P_rows['clicked'].to_numpy().astype(np.float32) + P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32) + bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32) + b = np.float32(np.log(y.mean() / (1 - y.mean()))) + gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6) + n = len(y) + for ep in range(epochs): + perm = rng.permutation(n) + for s in range(0, n, batch): + idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx] + pu, qi = P[ub], Q[ib] + z = b + bu[ub] + bi[ib] + (pu * qi).sum(1) + g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz + m = len(idx) + Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m)) + Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m)) + dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None] + dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None] + dbu = Su @ g; dbi = Si @ g + gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi + P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ) + bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi) + return dict(b=b, bu=bu, bi=bi, P=P, Q=Q) + +def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + g = float(P_rows['clicked'].mean()) + ndays = (C - J20) / DAY + CL = click_log(C, P_ev, P_rows, T_ev) + + # ---------------- item ---------------- + def ictr(c, n): + return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM) + it = P_rows.group_by('ii').agg( + pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'), + (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'), + (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6')) + it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'), + ictr('it_clk6', 'it_imp6').alias('it_ctr6')) + it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'), + (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'), + (pl.col('it_imp') / ndays).alias('it_imp_rate')) + ip = CL.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'), + (pl.col('ts') < J20).sum().alias('it_nclk_warm'), + ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d')) + + # ---------------- user ---------------- + ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'), + pl.col('ts').max().alias('_u_last_ev')) + ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr')) + uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'), + pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'), + (pl.col('ts') >= C - DAY).sum().alias('us_nclk24')) + uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate')) + + # ---------------- user x item ---------------- + uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'), + (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk')) + uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'), + pl.col('ts').max().alias('_ui_last_imp')) + uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk')) + + # ---------------- assemble ---------------- + R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns]) + .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left') + .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left')) + R = R.with_columns( + pl.col('h').list.len().alias('us_hlen'), + (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'), + pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'), + (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'), + (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'), + (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'), + (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'), + (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'), + pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt')) + # position (from end, 1 = most recent) of candidate in own history + hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii')) + .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own'))) + R = R.join(hx, on='r', how='left') + R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY), + pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY)) + R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in')) + + # ---------------- co-click ---------------- + co, trn = cooc_tables(CL, NI) + last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20')) + T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left') + X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos']) + .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32)) + .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left') + .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0))) + l5 = pl.col('pos') <= 5 + agg = X.group_by('r').agg( + pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'), + pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'), + pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'), + pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'), + pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'), + pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'), + (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20')) + R = R.join(agg, on='r', how='left').sort('r') + + # ---------------- SVD embeddings ---------------- + # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories; + # 'lag1d' = only clicks/impressions with ts < C - 1 day + if sv_mode == 'std': + CLsv, NEGsrc = CL, P_rows + elif sv_mode == 'notgt': + CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'), + _hist_long(P_ev).filter(pl.col('ts') < C)]).unique() + NEGsrc = P_rows + elif sv_mode == 'lag1d': + CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY) + else: + raise ValueError(sv_mode) + assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C + cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc + uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64) + ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos']) + # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway + er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32) + er_l5 = ex['pos'].to_numpy() <= 5 + nR = R.height + svcols = {} + for k in (16, 32, 64): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = (U * np.sqrt(S)).astype(np.float32) + Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9) + cand = Vn[iic] + svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1) + svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1) + hv = np.zeros((nR, k), np.float32); np.add.at(hv, er_r, Vn[er_j] * er_w[:, None]) + hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9) + if k == 32: + svcols['sv_k32_hnorm'] = hn + sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1) + mx = np.full(nR, -1.0, np.float32); np.maximum.at(mx, er_r, sims) + mx5 = np.full(nR, -1.0, np.float32); np.maximum.at(mx5, er_r[er_l5], sims[er_l5]) + svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5 + if k == 64: + # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only) + Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + neg = NEGsrc.filter(pl.col('clicked') == 0) + Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI)) + nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9 + nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9 + cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1) + svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn + for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')): + M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9 + svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()]) + + # ---------------- MF ---------------- + if with_mf: + mf = fit_mf(P_rows, NU, NI) + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + dot = (mf['P'][uu] * mf['Q'][ii]).sum(1) + R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot), + pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii])) + + fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] + + [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] + + [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)]) + + # ---------------- event context ---------------- + main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk', + 'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last', + 'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else []) + exprs = [] + for c in main: + x = pl.col(c).cast(pl.Float64) + exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'), + ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'), + x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')] + R = R.with_columns(exprs) + R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r') + log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s') + return R + +GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf'] + +def feature_cols(df, groups): + cols = [] + for c in df.columns: + if c in ID_COLS: continue + gname = c.split('_')[0] + if gname == 'ev': + src = c[3:].split('_')[0] # ev__... + if 'ev' in groups and src in groups: cols.append(c) + elif gname in groups: + cols.append(c) + return cols diff --git a/v12/sol_work/src12/v3/prep.py b/v12/sol_work/src12/v3/prep.py new file mode 100644 index 0000000000000000000000000000000000000000..d765f00826f7df7a80e886852b2c2571938b4eb6 --- /dev/null +++ b/v12/sol_work/src12/v3/prep.py @@ -0,0 +1,65 @@ +"""Parse train/evaluation CSVs into compact parquet tables (dense int ids). + +Outputs in v3/data/: + rows_train.parquet : rowid, eid, ii (item idx), clicked + rows_eval.parquet : rowid, eid, ii, ord (original row order) + ev_train.parquet / ev_eval.parquet : eid, u (user idx), ts, tb, dt, h (list[int item idx]), h_ts (list[int]) + items.parquet / users.parquet : id string -> idx +Usage: python prep.py [DATA_DIR] +""" +import polars as pl, sys, time, os +DATA = sys.argv[1] if len(sys.argv) > 1 else '@@RAW@@' +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') +os.makedirs(OUT, exist_ok=True) +t0 = time.time() +def log(*a): print(f'[{time.time()-t0:6.1f}s]', *a, flush=True) + +def scan(name): + return pl.scan_csv(f'{DATA}/{name}.csv', schema_overrides={'history_item_ids': pl.Utf8, 'history_timestamps': pl.Utf8, + 'rowid': pl.Utf8, 'event_id': pl.Utf8, 'user_id': pl.Utf8, 'item_id': pl.Utf8}) + +raw = {} +for name, lab in (('train', True), ('evaluation', False)): + lf = scan(name) + cols = ['rowid', 'event_id', 'item_id'] + (['clicked'] if lab else []) + rows = lf.select(cols).collect(engine='streaming') + ev = (lf.select('event_id', 'user_id', 'timestamp', 'time_bucket', 'day_type', 'history_item_ids', 'history_timestamps') + .unique(subset='event_id', keep='any') + .with_columns(pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).alias('h_str'), + pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts')) + .drop('history_item_ids', 'history_timestamps').collect(engine='streaming')) + raw[name] = (rows, ev) + log(name, rows.shape, ev.shape) + +tr_rows, tr_ev = raw['train']; ev_rows, ev_ev = raw['evaluation'] +items = pl.concat([tr_rows['item_id'], ev_rows['item_id'], tr_ev['h_str'].explode().drop_nulls(), + ev_ev['h_str'].explode().drop_nulls()]).unique().sort() +items = pl.DataFrame({'item_id': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)}) +users = pl.concat([tr_ev['user_id'], ev_ev['user_id']]).unique().sort() +users = pl.DataFrame({'user_id': users, 'u': pl.arange(0, len(users), eager=True).cast(pl.Int32)}) +events = pl.concat([tr_ev['event_id'], ev_ev['event_id']]).unique().sort() +events = pl.DataFrame({'event_id': events, 'eid': pl.arange(0, len(events), eager=True).cast(pl.Int32)}) +log('items', len(items), 'users', len(users), 'events', len(events)) +items.write_parquet(f'{OUT}/items.parquet'); users.write_parquet(f'{OUT}/users.parquet') + +def map_rows(rows, lab): + r = rows.with_row_index('ord').join(events, on='event_id', how='left').join(items, on='item_id', how='left') + sel = ['ord', 'rowid', 'eid', 'ii'] + (['clicked'] if lab else []) + r = r.sort('ord').select(sel) + if lab: r = r.with_columns(pl.col('clicked').cast(pl.Int8)) + return r + +def map_ev(ev): + e = ev.join(events, on='event_id', how='left').join(users, on='user_id', how='left') + ex = e.select('eid', 'h_str').with_columns(pl.int_ranges(0, pl.col('h_str').list.len()).alias('k')).explode(['h_str', 'k']).drop_nulls('h_str') + ex = ex.join(items.rename({'item_id': 'h_str'}), on='h_str', how='left').sort('eid', 'k') + h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').alias('h')) + e = e.join(h, on='eid', how='left').with_columns(pl.col('h').fill_null(pl.lit([], dtype=pl.List(pl.Int32)))) + return e.select('eid', 'u', pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'), + pl.col('day_type').cast(pl.Int8).alias('dt'), 'h', 'h_ts').sort('ts', 'eid') + +map_rows(tr_rows, True).drop('ord').write_parquet(f'{OUT}/rows_train.parquet') +map_rows(ev_rows, False).write_parquet(f'{OUT}/rows_eval.parquet') +map_ev(tr_ev).write_parquet(f'{OUT}/ev_train.parquet') +map_ev(ev_ev).write_parquet(f'{OUT}/ev_eval.parquet') +log('done') diff --git a/v12/sol_work/src12/v3/split.py b/v12/sol_work/src12/v3/split.py new file mode 100644 index 0000000000000000000000000000000000000000..7ec658f41a419f776d82b2bf76d97bc2ba3cd155 --- /dev/null +++ b/v12/sol_work/src12/v3/split.py @@ -0,0 +1,36 @@ +"""Time-based splits that mirror the official evaluation. + +Evaluation = each user's FIRST event at/after the cutoff. So for a cutoff C: + fit events = all train events with ts < C (labels usable for aggregates) + target/val = for each user, the first train event with C <= ts < C+W (W=None -> until end of train) +Whole events always stay together (split is by event, rows follow their event). +""" +import polars as pl + +J20 = 1737331200 # 2025-01-20 00:00 UTC (start of train window; earlier clicks = warm-up) +DAY = 86400 +C_MAIN = J20 + 5 * DAY # 2025-01-25 +C_SECOND = J20 + 6 * DAY # 2025-01-26 +C_FINAL = J20 + 7 * DAY # 2025-01-27 (official cutoff) + +def fit_events(TE: pl.DataFrame, C: int) -> pl.DataFrame: + return TE.filter(pl.col('ts') < C) + +def first_events(TE: pl.DataFrame, C: int, W: int | None = None) -> pl.DataFrame: + """First event per user in [C, C+W). Ties on ts broken by eid (deterministic).""" + m = pl.col('ts') >= C + if W is not None: + m = m & (pl.col('ts') < C + W) + return TE.filter(m).sort('ts', 'eid').group_by('u', maintain_order=True).first() + +def split(TE: pl.DataFrame, TR: pl.DataFrame, C: int, W: int | None = None): + """Return (fit_ev, fit_rows, val_ev, val_rows).""" + fe = fit_events(TE, C) + ve = first_events(TE, C, W) + fr = TR.join(fe.select('eid'), on='eid', how='semi') + vr = TR.join(ve.select('eid'), on='eid', how='semi') + return fe, fr, ve, vr + +def train_fold_cutoffs(C_val: int): + """Daily cutoffs strictly before C_val used to build model-training rows (each fold: first event/user in [Ck, Ck+1d)).""" + return [J20 + k * DAY for k in range(1, (C_val - J20) // DAY)] diff --git a/v12/sol_work/src12/v3feat_ne.py b/v12/sol_work/src12/v3feat_ne.py new file mode 100644 index 0000000000000000000000000000000000000000..c74e8e38d1294e2825fbc443990833bb59752b67 --- /dev/null +++ b/v12/sol_work/src12/v3feat_ne.py @@ -0,0 +1,298 @@ +"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR. + +build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate. +Every statistic uses ONLY: + * train events/rows with ts < C (fit window: impressions + labels), and + * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts). +Since target events are each user's first event at/after C, their histories contain only clicks < C, +so nothing observed after a target row's own event time is ever joined. + +Feature groups (column prefixes, used for ablation): + hist_ candidate vs the user's click log / own history / past impressions of that user-item pair + it_ item popularity & CTR (full fit window, last 24h, last 6h, trend) + co_ item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks + us_ user activity / CTR / recency / cold-start flags + ev_ within-event rank / z-score / event max & mean of the main features + tm_ time_bucket, day_type, hour, staleness since cutoff + sv_ truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines + mf_ logistic matrix factorisation score fitted on fit-window impressions +""" +import time +import numpy as np +import polars as pl +from scipy import sparse +from sklearn.utils.extmath import randomized_svd + +J20 = 1737331200 +DAY = 86400 +G_PRIOR = 0.0629 +K_ITEM = 20 # pseudo-impressions for item CTR shrinkage +K_USER = 5 # pseudo-events for user CTR shrinkage +TOPK = 200 # co-click neighbours kept per item +ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord'] +_t0 = time.time() + +def log(*a): + print(time.strftime('%H:%M:%S'), *a, flush=True) + +def load(D): + TE = pl.read_parquet(f'{D}/ev_train.parquet') + TR = pl.read_parquet(f'{D}/rows_train.parquet') + EE = pl.read_parquet(f'{D}/ev_eval.parquet') + ER = pl.read_parquet(f'{D}/rows_eval.parquet') + NI = pl.read_parquet(f'{D}/items.parquet').height + NU = pl.read_parquet(f'{D}/users.parquet').height + return TE, TR, EE, ER, NI, NU + +def _hist_long(E): + return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h') + .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts'))) + +def click_log(C, P_ev, P_rows, T_ev): + """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories.""" + win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts') + hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C) + CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii') + return CL + +def cooc_tables(CL, NI): + """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next.""" + us = CL.select('u', 'ii').unique() + pop = us.group_by('ii').agg(pl.len().alias('pop')) + pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b')) + .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co'))) + co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b') + .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos')) + .select(pl.col('ii_b').alias('j'), 'ii', 'cos') # j = history item, ii = candidate + .sort('cos', descending=True).group_by('j').head(TOPK)) + seq = CL.select('u', 'ii') # already sorted by u, ts + parts = [] + for lag in (1, 2, 3): + parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j') + .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w'))) + trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum()) + tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot')) + trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp')) + .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK)) + return co, trn + +def _segmax(n, idx, vals, init=-1.0): + """v8: == np.maximum.at(np.full(n, init), idx, vals), via sort + reduceat.""" + out = np.full(n, init, np.float32) + if len(idx) == 0: return out + o = np.argsort(idx, kind='stable'); si = idx[o]; sv = vals[o].astype(np.float32) + st = np.flatnonzero(np.r_[True, si[1:] != si[:-1]]) + out[si[st]] = np.maximum(out[si[st]], np.maximum.reduceat(sv, st)); return out + +def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000): + """Logistic MF logit = b + bu + bi + fitted with Adagrad on fit-window impressions (numpy).""" + rng = np.random.default_rng(seed) + u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64) + y = P_rows['clicked'].to_numpy().astype(np.float32) + P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32) + bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32) + b = np.float32(np.log(y.mean() / (1 - y.mean()))) + gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6) + n = len(y) + for ep in range(epochs): + perm = rng.permutation(n) + for s in range(0, n, batch): + idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx] + pu, qi = P[ub], Q[ib] + z = b + bu[ub] + bi[ib] + (pu * qi).sum(1) + g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32) # dL/dz + m = len(idx) + Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m)) + Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m)) + dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None] + dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None] + dbu = Su @ g; dbi = Si @ g + gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi + P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ) + bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi) + return dict(b=b, bu=bu, bi=bi, P=P, Q=Q) + +def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'): + t0 = time.time() + P_ev = TE.filter(pl.col('ts') < C) + P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner') + g = float(P_rows['clicked'].mean()) + ndays = (C - J20) / DAY + CL = click_log(C, P_ev, P_rows, T_ev) # per-user stats: train data + each target row's OWN history + CLg = click_log(C, P_ev, P_rows, T_ev.head(0)) # item-level / global stats: train data only (no target-event histories) + + # ---------------- item ---------------- + def ictr(c, n): + return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM) + it = P_rows.group_by('ii').agg( + pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'), + (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'), + (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6')) + it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'), + ictr('it_clk6', 'it_imp6').alias('it_ctr6')) + it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'), + (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'), + (pl.col('it_imp') / ndays).alias('it_imp_rate')) + ip = CLg.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'), + (pl.col('ts') < J20).sum().alias('it_nclk_warm'), + ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d')) + + # ---------------- user ---------------- + ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'), + pl.col('ts').max().alias('_u_last_ev')) + ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr')) + uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'), + pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'), + (pl.col('ts') >= C - DAY).sum().alias('us_nclk24')) + uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate')) + + # ---------------- user x item ---------------- + uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'), + (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk')) + uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'), + pl.col('ts').max().alias('_ui_last_imp')) + uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk')) + + # ---------------- assemble ---------------- + R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns]) + .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r')) + R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left') + .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left')) + R = R.with_columns( + pl.col('h').list.len().alias('us_hlen'), + (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'), + pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'), + (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'), + (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'), + (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'), + (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'), + (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'), + pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt')) + # position (from end, 1 = most recent) of candidate in own history + hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos')) + .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii')) + .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own'))) + R = R.join(hx, on='r', how='left') + R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY), + pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY)) + R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in')) + + # ---------------- co-click ---------------- + co, trn = cooc_tables(CLg, NI) + last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20')) + T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left') + X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos']) + .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32)) + .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left') + .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0))) + l5 = pl.col('pos') <= 5 + agg = X.group_by('r').agg( + pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'), + pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'), + pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'), + pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'), + pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'), + pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'), + (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20')) + R = R.join(agg, on='r', how='left').sort('r') + + # ---------------- SVD embeddings ---------------- + # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories; + # 'lag1d' = only clicks/impressions with ts < C - 1 day + if sv_mode == 'std': + CLsv, NEGsrc = CLg, P_rows + elif sv_mode == 'notgt': + CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'), + _hist_long(P_ev).filter(pl.col('ts') < C)]).unique() + NEGsrc = P_rows + elif sv_mode == 'lag1d': + CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY) + else: + raise ValueError(sv_mode) + assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C + cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n')) + Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32) + Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc + # user-side matrix incl. each target user's own history (fold-in onto item factors learned from train-only clicks) + cnt = CL.filter(pl.col('ts') < C).group_by('u', 'ii').agg(pl.len().alias('n')) if sv_mode == 'std' else cnt + Xu = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) @ sparse.diags(idf) + Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu + uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64) + ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos']) + # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway + er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32) + er_l5 = ex['pos'].to_numpy() <= 5 + nR = R.height + svcols = {} + Werj = sparse.csr_matrix((er_w, (er_r, er_j)), shape=(nR, NI)) + for k in (16, 32, 64): + U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5) + V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = np.asarray(Xu @ (Vt.T / np.sqrt(S))).astype(np.float32) # fold-in = U*sqrt(S) for train users + Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9) + cand = Vn[iic] + svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1) + svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1) + hv = np.asarray(Werj @ Vn, dtype=np.float32) # v8: sparse matmul replaces np.add.at + hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9) + if k == 32: + svcols['sv_k32_hnorm'] = hn + sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1) + mx = _segmax(nR, er_r, sims); mx5 = _segmax(nR, er_r[er_l5], sims[er_l5]) + svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5 + if k == 64: + # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only) + Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) + neg = NEGsrc.filter(pl.col('clicked') == 0) + Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI)) + nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9 + nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9 + cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1) + svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn + for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')): + M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9 + svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1) + R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()]) + + # ---------------- MF ---------------- + if with_mf: + mf = fit_mf(P_rows, NU, NI) + uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64) + dot = (mf['P'][uu] * mf['Q'][ii]).sum(1) + R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot), + pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii])) + + fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List] + R = R.with_columns([pl.col(c).fill_null(0) for c in fill0]) + R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] + + [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] + + [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)]) + + # ---------------- event context ---------------- + main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk', + 'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last', + 'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else []) + exprs = [] + for c in main: + x = pl.col(c).cast(pl.Float64) + exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'), + ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'), + x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')] + R = R.with_columns(exprs) + R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r') + log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s') + return R + +GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf'] + +def feature_cols(df, groups): + cols = [] + for c in df.columns: + if c in ID_COLS: continue + gname = c.split('_')[0] + if gname == 'ev': + src = c[3:].split('_')[0] # ev__... + if 'ev' in groups and src in groups: cols.append(c) + elif gname in groups: + cols.append(c) + return cols diff --git a/v12/sol_work/src12/v3fold.py b/v12/sol_work/src12/v3fold.py new file mode 100644 index 0000000000000000000000000000000000000000..a65c7ac661cea81ee94a30e1026848530fbb0777 --- /dev/null +++ b/v12/sol_work/src12/v3fold.py @@ -0,0 +1,18 @@ +"""v3 features (v2work/v3feat_ne.py = v3/features.py with global/item stats from train data only) for all fold specs. +Usage: python v3fold.py NAME... -> v2work/feat_v3/NAME.parquet + f1..f6: first event/user in [Ck, Ck+1d); val25/val26: [C, end of train); eval: Jan27 cutoff + e1..e4: [Ck, Jan25) (eval-like staleness, for cutoff-25 validation); E1..E6: [Ck, Jan27) (eval-like, final fit)""" +import sys, os +sys.path.insert(0, '@@WORK@@'); sys.path.insert(1, '@@WORK@@/v3') +from v3feat_ne import load, build, log +from split import J20, DAY, first_events +TE, TR, EE, ER, NI, NU = load('@@WORK@@/v3/data') +OUT = '@@WORK@@/feat_v3'; os.makedirs(OUT, exist_ok=True) +SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), + **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}, **{f'd{k}': (J20 + k * DAY, (6 - k) * DAY) for k in range(1, 6)}} +for n in sys.argv[1:]: + if n == 'eval': R = build(J20 + 7 * DAY, EE, ER, TE, TR, NI, NU) + else: + C, W = SPEC[n]; T_ev = first_events(TE, C, W) + R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU) + R.write_parquet(f'{OUT}/{n}.parquet'); log('wrote', n, R.shape) diff --git a/v12/sol_work/src12/validate.py b/v12/sol_work/src12/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..09008b3ba2c2b62b2d43eb6e2f1c3ee931320ff0 --- /dev/null +++ b/v12/sol_work/src12/validate.py @@ -0,0 +1,18 @@ +"""Standalone validator: python validate.py SUB.csv""" +import sys, csv, math, numpy as np, polars as pl +out=sys.argv[1] +ids=pl.read_csv('@@WORK@@/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8})['rowid'].to_list() +raw=open(out,'rb').read(); errs=[] +try: raw.decode('utf-8') +except Exception: errs.append('not utf-8') +with open(out,newline='',encoding='utf-8') as f: + r=csv.reader(f); h=next(r); rows=list(r) +if h!=['rowid','predicted_probability']: errs.append(f'header {h}') +if len(rows)!=529390: errs.append(f'rows {len(rows)}') +sid=[x[0] for x in rows] +if len(set(sid))!=len(sid): errs.append('dup rowids') +if set(sid)!=set(ids): errs.append('rowid set mismatch') +v=np.array([float(x[1]) for x in rows]) +if not (np.isfinite(v).all() and (v>=0).all() and (v<=1).all()): errs.append('bad values') +print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.5g} max={v.max():.4g} distinct={len(np.unique(v))}') +print('VALID' if not errs else 'INVALID '+';'.join(errs)) diff --git a/v12/sol_work/src12/write_sub.py b/v12/sol_work/src12/write_sub.py new file mode 100644 index 0000000000000000000000000000000000000000..8852ed160597cce378b39fcbb9b5ab1bf43914bc --- /dev/null +++ b/v12/sol_work/src12/write_sub.py @@ -0,0 +1,27 @@ +"""Write + validate submission from eval_pred.parquet. Usage: python write_sub.py OUT.csv""" +import sys, csv, math, hashlib, zipfile, os, numpy as np, polars as pl +out=sys.argv[1] +pr=pl.read_parquet(os.environ.get('PRED','@@WORK@@/eval_pred.parquet')) +ev=pl.read_csv('@@WORK@@/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8}) +df=ev.join(pr,on='rowid',how='left',maintain_order='left') +p=np.nan_to_num(df['p'].to_numpy().astype(np.float64),nan=0.0629,posinf=1.0,neginf=0.0).clip(0,1) +p[np.isnan(df['p'].to_numpy().astype(float))]=0.0629 +print('missing preds filled:',int(df['p'].null_count())) +with open(out,'w',encoding='utf-8',newline='') as f: + f.write('rowid,predicted_probability\n'); f.writelines(f'{r},{v:.8g}\n' for r,v in zip(df['rowid'].to_list(),p)) +# ---- validator +ids=ev['rowid'].to_list(); errs=[] +with open(out,newline='',encoding='utf-8') as f: + r=csv.reader(f); h=next(r); rows=list(r) +if h!=['rowid','predicted_probability']: errs.append('header') +if len(rows)!=529390 or len(ids)!=529390: errs.append(f'rows {len(rows)}') +sid=[x[0] for x in rows] +if len(set(sid))!=len(sid) or set(sid)!=set(ids): errs.append('rowid set mismatch') +vals=[float(x[1]) for x in rows] +if not all(math.isfinite(v) and 0<=v<=1 for v in vals): errs.append('bad values') +v=np.array(vals); print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.4g} max={v.max():.4g} distinct={len(np.unique(v))}') +print('VALID' if not errs else 'INVALID '+';'.join(errs)) +zp=out+'.zip' +with zipfile.ZipFile(zp,'w',zipfile.ZIP_DEFLATED) as z: z.write(out,arcname=os.path.basename(out)) +with zipfile.ZipFile(zp) as z: print('zip contents',z.namelist()) +print('sha256',hashlib.sha256(open(out,'rb').read()).hexdigest()) diff --git a/v12/sol_work/tail12.txt b/v12/sol_work/tail12.txt new file mode 100644 index 0000000000000000000000000000000000000000..5959786c4a44bf1305241d9497e8d2c96b355fa9 --- /dev/null +++ b/v12/sol_work/tail12.txt @@ -0,0 +1,54 @@ + + +SLOW = '"learning_rate":0.03,"feature_fraction":0.55,"num_leaves":95,"lambda_l2":30' +BIG = '"learning_rate":0.015,"num_leaves":127,"feature_fraction":0.5,"lambda_l2":30,"min_data_in_leaf":200,"max_bin":255,"force_col_wise":true' +LP = '"max_bin":63,"force_col_wise":true' +LR = 'feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw' +DROP = "'^sq_(pdot|pcos|mdot|mcos)$'" +def pipeline(W, RAW, T, PY, OUT, cut): + """ordered (stage, shell command or None for setup, subdir of W)""" + FOLDS = 'f3 f4 f5 f6 eval' if cut == 27 else 'f1 f2 f3 f4 val25' + V2 = 'lite' if cut == 27 else 'c25' + return [('setup', None, ''), ('prep_v3', f'{PY} prep.py {RAW}', 'v3'), ('prep_v2', f'{PY} prep.py', ''), + ('feat_v3', f'{PY} v3fold.py {FOLDS}', ''), ('feat_v2', f'{PY} build_folds.py {V2}', ''), + ('feat_x', f'{PY} extra.py {FOLDS}', ''), ('feat_lr', f'LR_KS=4,6,8,10,12,16 LR_CON=1 {PY} lrfeat2.py {FOLDS}', ''), + ('feat_tt', f'TT_THREADS={T} {PY} ttfeat.py {FOLDS}', ''), ('feat_pp', f'{PY} ppmi.py {FOLDS}', ''), + ('feat_sq', f'V12_THREADS={T} {PY} seqrun.py {FOLDS}', ''), ('feat_dw', f'{PY} dw/mkdw.py {FOLDS}', ''), + ('dwbig', f"{PY} ens.py {cut} dwbig --seeds 0 --rounds 1000 --save 1000 --threads {T} --lr {LR} --nfolds 4 --drop {DROP} --params '{{{BIG}}}'", ''), + ('dw_s', f"{PY} ens.py {cut} dw_s --seeds 0 --rounds 500 --save 500 --threads {T} --lr {LR} --nfolds 4 --negfrac 0.5 --drop {DROP} --params '{{{SLOW},{LP}}}'", ''), + *[(f'mlp{s}', f'MLP_THREADS={T} {PY} mlp12.py {cut} dwmlp --ep 4 --seed {s} --lr {LR}', '') for s in range(3)], + ('blend', f'{PY} blend12.py {cut}', ''), + *([('write', f'PRED={W}/eval_pred.parquet {PY} write_sub.py {OUT}', ''), ('validate', f'{PY} validate.py {OUT}', '')] if cut == 27 else [])] +def setup(W, RAW): + os.makedirs(os.path.join(W, 'v3'), exist_ok=True); os.makedirs(os.path.join(W, 'ens'), exist_ok=True) + for k, s in SRC.items(): + os.makedirs(os.path.dirname(os.path.join(W, k)), exist_ok=True) + open(os.path.join(W, k), 'w').write(s.replace('@@WORK@@', W).replace('@@RAW@@', RAW)) + for f in ('train.csv', 'evaluation.csv'): # v2 prep / writer read the raw CSVs from WORK + dst = os.path.join(W, f) + if not os.path.exists(dst): os.symlink(os.path.join(RAW, f), dst) +def thread_env(T): + return dict(os.environ, POLARS_MAX_THREADS=T, OMP_NUM_THREADS=T, OPENBLAS_NUM_THREADS=T, MKL_NUM_THREADS=T, NUMEXPR_NUM_THREADS=T, + TT_THREADS=T, SQ_THREADS=T, RAYON_NUM_THREADS=T, PYTHONUNBUFFERED='1') +def main(): + ap = argparse.ArgumentParser(); ap.add_argument('--raw', required=True); ap.add_argument('--work', default='./v12_work') + ap.add_argument('--threads', type=int, default=8); ap.add_argument('--out', default=None); ap.add_argument('--from', dest='frm', default='setup'); ap.add_argument('--to', default=None) + ap.add_argument('--cut', type=int, default=27, choices=(25, 27)) + a = ap.parse_args() + W = os.path.abspath(a.work); RAW = os.path.abspath(a.raw); T = str(a.threads) + OUT = os.path.abspath(a.out or os.path.join(W, 'submission_v12.csv')) + os.makedirs(os.path.dirname(OUT), exist_ok=True) + env = thread_env(T); PY = sys.executable + P = pipeline(W, RAW, T, PY, OUT, a.cut); names = [p[0] for p in P]; to = a.to or names[-1] + tf = os.path.join(W, 'timings.json'); tim = json.load(open(tf)) if os.path.exists(tf) and a.frm != 'setup' else {} + t_all = time.time() + for st, cmd, sub in P[names.index(a.frm):names.index(to) + 1]: + t0 = time.time() + if cmd is None: setup(W, RAW) + else: print('>>', cmd, flush=True); subprocess.run(cmd, shell=True, check=True, cwd=os.path.join(W, sub), env=env) + tim[st] = round(time.time() - t0, 1); print(f'[stage {st}] {tim[st]:.0f}s', flush=True) + os.makedirs(W, exist_ok=True); json.dump(tim, open(tf, 'w'), indent=1) + print('TOTAL this run %.1f min; all stages %.1f min' % ((time.time() - t_all) / 60, sum(tim.values()) / 60)) + if a.cut == 27: print('submission:', OUT) +if __name__ == '__main__': + main() diff --git a/v12/sol_work/verify_stats_c25.json b/v12/sol_work/verify_stats_c25.json new file mode 100644 index 0000000000000000000000000000000000000000..37b6e4e524a4bfacf8723a6d85ce7faaaedf760b --- /dev/null +++ b/v12/sol_work/verify_stats_c25.json @@ -0,0 +1,31 @@ +{ + "cut": 25, + "local_auc": { + "dwbig": 0.81661, + "dw_s": 0.81611, + "mlp": 0.81646, + "blend": 0.81727 + }, + "v12_ref_blend": 0.81723, + "delta": 4e-05, + "stage_seconds": { + "setup": 0.0, + "prep_v3": 33.9, + "prep_v2": 7.8, + "feat_v3": 113.0, + "feat_v2": 55.6, + "feat_x": 70.9, + "feat_lr": 15.6, + "feat_tt": 35.3, + "feat_pp": 34.4, + "feat_sq": 1799.9, + "feat_dw": 304.5, + "dwbig": 302.4, + "dw_s": 80.6, + "mlp0": 481.7, + "mlp1": 97.7, + "mlp2": 647.0, + "blend": 2.5 + }, + "note": "clean-dir run of solution_v12.py --cut 25 (raw CSVs only); mlp stages slowed by concurrent DW jobs" +} \ No newline at end of file diff --git a/v12x/NOTES.md b/v12x/NOTES.md new file mode 100644 index 0000000000000000000000000000000000000000..c610329824c49a3d09b87c528bfcb1ff5c617098 --- /dev/null +++ b/v12x/NOTES.md @@ -0,0 +1,19 @@ +# v12x NOTES (XGBoost / lambdarank / max_bin experiments on v12 features) +- Setup: xgboost 3.4.1 in v12x/pylib (uv --target; main venv untouched). xens.py = XGB on v12 matrix (435 cols after dropping sq_(pdot|pcos|mdot|mcos); the "562" quoted by parent = 435 used by dwbig; same cl.matrix, same folds). cl.py = copy of v12/cl.py that globs the base cache by table name (original key includes file mtimes, changed by the snapshot restore). +-inf clipped to +-1e15 (xgboost rejects inf). +- XGB params: hist, lossguide max_leaves127, eta .03, subsample .8, colsample_bytree .5, min_child_weight 30, lambda 30, max_bin 255. +- [04:12] Round pick on C25 train folds f1-f3, early stop on hold fold f4 (not val): best_iter 403 (hold AUC .80985), 553 s. Declared: use 450 rounds for all cuts (saved 300/450/600). +- [04:12] c25 clean solution run: blend .81727 (see v12/sol_work/verify_stats_c25.json). +- [04:15] queued: run_xgb.sh (xgb c25/c23 seeds 0,1 @450 rounds), then run_q2.sh (lgb255 sanity, lambdarank, rank_xendcg, max_bin 1023/511; c25+c23). logs in v12x/logs, results ens/xlog.jsonl, timings ens/timings.jsonl +- [04:15] XGB c25 s0: @300 .81605 @450 .81619 @600 .81622 (dwbig .81658); fit 368s. Parent queued: combined v12+propensity+tm_prod retrain (+ablations, + 'in-history x taste' interaction) after q2. +- [04:35] built per-fold features with v12x/mk/mkfeat.py (f1-f4,val23,val25): feat_prop (28; matches clean25 file except ~110 tie rows), feat_tm (tm_prod0-11+tail; matches DW cache val25 1e-4), feat_tmx (tm_inh,tm_cos,_in,_out,_dm,tm_inh_n interaction). cl.py now prefers v12x/ for extra feature dirs. +- [04:40] queued run_q3.sh: dwbig + {prop+tm, tm, prop, tm+tmx, all} on C25,C23 (tags f_*). + +## 04:32 XGB result (hist CPU, lossguide 127 leaves, eta .03, ss .8, cs .5, mcw 30, lambda 30, 600 rounds; primary = @450 from early-stop pick best_iter 403 on f4 hold) +C25 alone: s0 .81619 / s1 .81634 (@450) vs dwbig .81658 / .81652 (gap -.0004 / -.0002; within noise band) +C23 alone: s0 .79424 / s1 .79399 (@450) vs dwbig .7946 / .79458 (gap -.0004 / -.0006) +Fit times: C25 ~6.7 min, C23 ~3.6 min (8 cores). +4th member, weights 1:1:2:w (@450): C25 w=.5/1/2: s0 .81726/.81725/.81721, s1 .81728/.81729/.81726 (v12 .81723) + C23: s0 .79572/.79573/.79568, s1 .79568/.79567/.79558 (v12 .79566) +Max delta w=1: C25 +.00002/+.00006, C23 +.00007/+.00001 => noise; FAILS accept rule. No v12x submission built. +## 04:34 queue q2/q3 killed (lrank C25 just started, lgb255 done) for priority p33 fingerprint + XGB probe job; restart later from lrank +## queued at END: CatBoost experiment /workspace/huawei/v15cat (parent msg 04:37) diff --git a/v12x/analyze.py b/v12x/analyze.py new file mode 100644 index 0000000000000000000000000000000000000000..b5ab4b41bc059be0c4c1ecbdf81e6a93a83b488b --- /dev/null +++ b/v12x/analyze.py @@ -0,0 +1,14 @@ +import sys, json, numpy as np, polars as pl, itertools, os +sys.path.insert(0,'/workspace/huawei/v12x'); sys.path.insert(1,'/workspace/huawei/v12x/pylib') +from sklearn.metrics import roc_auc_score +V='/workspace/huawei/v12/ens'; X='/workspace/huawei/v12x/ens' +y={c:pl.read_parquet(f'/workspace/huawei/v12/feat_v3/val{c}.parquet',columns=['clicked'])['clicked'].to_numpy() for c in (25,23)} +def ld(p,col=0): return np.load(p)[:,col].astype(np.float64) +def auc(c,s): return round(roc_auc_score(y[c],s),5) +def base(c): + big=ld(f'{V}/dwbig_c{c}_s0.npy',1); sm=ld(f'{V}/dw_s_c{c}_s0.npy',1) if c==25 else ld(f'{V}/dw_s_c{c}_s0.npy',1) + ml=np.mean([ld(f'{V}/dwmlp_c{c}_s{s}.npy',3) for s in (0,1,2)],0); return big,sm,ml +def demean(s): g=s.reshape(-1,10); return (g-g.mean(1,keepdims=True)).ravel() +if __name__=='__main__': + for c in (25,23): + big,sm,ml=base(c); z=(big+sm+2*ml)/4; print('cut',c,'dwbig',auc(c,big),'dw_s',auc(c,sm),'mlp',auc(c,ml),'v12 blend',auc(c,z)) diff --git a/v12x/cl.py b/v12x/cl.py new file mode 100644 index 0000000000000000000000000000000000000000..16123cf36e0301c7be06b9ef8f5136266b4ccdcf --- /dev/null +++ b/v12x/cl.py @@ -0,0 +1,65 @@ +"""Shared loader + feature-matrix cache (step C). Matrices cached as parquet keyed by hash(table names, feature list, sample).""" +import os, sys, hashlib, json, gc, numpy as np, polars as pl +sys.path.insert(0,'/workspace/huawei/v9'); sys.path.insert(1,'/workspace/huawei/v3') +from feats import DROP +from features import feature_cols, GROUPS +V9='/workspace/huawei/v9'; D='/workspace/huawei/v12'; P2=f'{V9}/feat'; P3=f'{V9}/feat_v3'; CACHE='/workspace/huawei/v9d/val/cache_mx' # read-only base cache +V2_OF={'val23':'val23','f1':'h1','f2':'h2','f3':'h3','f4':'h4','f5':'h5','f6':'val26','val25':'val25','val26':'val26','eval':'eval'} +V2_OF.update({f'{p}{k}':f'{p}{k}' for p in 'eEd' for k in range(1,7)}) +TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour'] +KEEP=['rowid','eid','u','ts','clicked','us_cold','us_nohist','tm_stale'] +def load_tab(n, extra=(), variant='notime'): + a=pl.read_parquet(f'{P3}/{n}.parquet'); b=pl.read_parquet(f'{P2}/{V2_OF[n]}.parquet') + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.read_parquet(f'{D}/{sub}/{n}.parquet'),on='rowid',how='left') + if '+' in variant: + import mods; a=mods.apply(a,n,variant) + return a +def featlist(df, variant='notime'): + f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith(('v2_','x_','r_','s_','ev2_','rc_','sh_','hs_','lr','tA_','tB_','uw_','la_','sq_'))] + if 'notime' in variant: f=[c for c in f if c not in TIME] + if variant.split('+')[0]=='stale': f=[c for c in f if c not in TIME or c=='tm_stale'] + return f +def usample(df, frac): + if frac>=1: return df + return df.filter((pl.col('u').hash(seed=7) % 10000) < int(frac*10000)) +FL=[None] +def load_tab_rows(n, rid, extra=(), variant='notime'): + """same as load_tab but restricted to rowids rid (polars Series) -> low memory""" + a=pl.scan_parquet(f'{P3}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + b=pl.scan_parquet(f'{P2}/{V2_OF[n]}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.scan_parquet(f'{D}/{sub}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect(),on='rowid',how='left') + return a +def build_chunked(n, extra, variant, frac, nch=4): + rid_all=pl.read_parquet(f'{P3}/{n}.parquet',columns=['rowid'])['rowid']; parts=[] + step=(len(rid_all)+nch-1)//nch + for i in range(nch): + rid=rid_all.slice(i*step,step); t=usample(load_tab_rows(n,rid,extra,variant),frac) + # restore original order within chunk + t=pl.DataFrame({'rowid':rid}).join(t,on='rowid',how='inner',maintain_order='left') + fl=featlist(t,variant); FL[0]=fl + t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl))) + t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP]) + parts.append(t); gc.collect() + return pl.concat(parts) +BASE=('feat_lr2','feat_tt','feat_pp') +def matrix(n, frac=1.0, variant='notime', extra=()): + """v9: base = cached v8-lite matrix (read-only, from v9d/val/cache_mx); new dirs (feat_tm*) joined on rowid. + returns (X float32, meta DataFrame, feats).""" + base=tuple(e for e in extra if e in BASE); new=[e for e in extra if e not in BASE]; assert base==BASE, extra + key=hashlib.md5(json.dumps([n,frac,variant,list(base),os.path.getmtime(f'{P3}/{n}.parquet')]).encode()).hexdigest()[:12] + import glob; _g=glob.glob(f'{CACHE}/{n}_*.parquet'); assert len(_g)==1,(n,_g); fp=_g[0] # v12x: key includes mtimes (changed by snapshot restore) -> pick the unique cache by table name + fl=json.load(open(fp+'.json')); t=pl.read_parquet(fp) + for sub in new: + e=pl.read_parquet((f'/workspace/huawei/v12x/{sub}' if os.path.isdir(f'/workspace/huawei/v12x/{sub}') else f'{D}/{sub}')+f'/{n}.parquet'); nc=[c for c in e.columns if c!='rowid' and c not in fl] + t=t.join(e.select(['rowid']+nc),on='rowid',how='left',maintain_order='left'); fl=fl+nc + drop=os.environ.get('V9DROP') + if drop: + import re; fl=[c for c in fl if not re.search(drop,c)] + X=t.select(fl).cast(pl.Float32).to_numpy(); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect() + return X, meta, fl diff --git a/v12x/logs/mkfeat.log b/v12x/logs/mkfeat.log new file mode 100644 index 0000000000000000000000000000000000000000..3bbcbbd60bf32c60edee8fba41eaa5b7148e955e --- /dev/null +++ b/v12x/logs/mkfeat.log @@ -0,0 +1,9 @@ +[ 5s] data loaded +/workspace/huawei/v12x/mk/mkfeat.py:39: RuntimeWarning: Mean of empty slice + 'tm_cos_dm':(g-np.nanmean(g,1,keepdims=True)).ravel().astype(np.float32),'tm_inh_n':np.repeat(inh.reshape(-1,10).sum(1),10)} +val25 (439640, 29) (439640, 14) (439640, 7) 6 s +val23 (501550, 29) (501550, 14) (501550, 7) 7 s +f1 (351560, 29) (351560, 14) (351560, 7) 5 s +f2 (351120, 29) (351120, 14) (351120, 7) 6 s +f3 (349120, 29) (349120, 14) (349120, 7) 5 s +f4 (350400, 29) (350400, 14) (350400, 7) 5 s diff --git a/v12x/logs/pick25.log b/v12x/logs/pick25.log new file mode 100644 index 0000000000000000000000000000000000000000..fcd20e24e5cc546daf5d216eff7f90275e57848a --- /dev/null +++ b/v12x/logs/pick25.log @@ -0,0 +1,13 @@ +[0] hold-auc:0.79660 +[50] hold-auc:0.80618 +[100] hold-auc:0.80795 +[150] hold-auc:0.80887 +[200] hold-auc:0.80934 +[250] hold-auc:0.80966 +[300] hold-auc:0.80972 +[350] hold-auc:0.80976 +[400] hold-auc:0.80983 +[450] hold-auc:0.80979 +[500] hold-auc:0.80973 +[503] hold-auc:0.80973 +{"pick": true, "cut": 25, "train": ["f1", "f2", "f3"], "hold": "f4", "best_iter": 403, "best_auc": 0.8098451883665725, "fit_s": 553} diff --git a/v12x/logs/run_q2.log b/v12x/logs/run_q2.log new file mode 100644 index 0000000000000000000000000000000000000000..57674ff5541008c7f4a7702fc5cfac9120889400 --- /dev/null +++ b/v12x/logs/run_q2.log @@ -0,0 +1 @@ +{"cut": 25, "tag": "lgb255", "seed": 0, "n": 1402200, "nf": 435, "load_s": 8, "ds_s": 9, "fit_s": 194, "min": 3.9, "obj": "binary", "max_bin": 255, "auc": {"800": 0.81647, "1000": 0.81658}} diff --git a/v12x/logs/run_q3.log b/v12x/logs/run_q3.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/v12x/logs/run_xgb.log b/v12x/logs/run_xgb.log new file mode 100644 index 0000000000000000000000000000000000000000..123d50aa1c7fce1084c3200436e130ed7140ed44 --- /dev/null +++ b/v12x/logs/run_xgb.log @@ -0,0 +1,5 @@ +{"cut": 25, "tag": "xgb", "seed": 0, "n": 1402200, "nf": 435, "load_s": 9, "fit_s": 368, "min": 6.8, "params": {"objective": "binary:logistic", "tree_method": "hist", "grow_policy": "lossguide", "max_leaves": 127, "max_depth": 0, "eta": 0.03, "subsample": 0.8, "colsample_bytree": 0.5, "min_child_weight": 30, "reg_lambda": 30, "max_bin": 255, "seed": 42, "eval_metric": "auc"}, "auc": {"300": 0.81605, "450": 0.81619, "600": 0.81622}} +{"cut": 23, "tag": "xgb", "seed": 0, "n": 702680, "nf": 435, "load_s": 4, "fit_s": 197, "min": 3.6, "params": {"objective": "binary:logistic", "tree_method": "hist", "grow_policy": "lossguide", "max_leaves": 127, "max_depth": 0, "eta": 0.03, "subsample": 0.8, "colsample_bytree": 0.5, "min_child_weight": 30, "reg_lambda": 30, "max_bin": 255, "seed": 42, "eval_metric": "auc"}, "auc": {"300": 0.79439, "450": 0.79424, "600": 0.79377}} +{"cut": 25, "tag": "xgb", "seed": 1, "n": 1402200, "nf": 435, "load_s": 9, "fit_s": 355, "min": 6.6, "params": {"objective": "binary:logistic", "tree_method": "hist", "grow_policy": "lossguide", "max_leaves": 127, "max_depth": 0, "eta": 0.03, "subsample": 0.8, "colsample_bytree": 0.5, "min_child_weight": 30, "reg_lambda": 30, "max_bin": 255, "seed": 43, "eval_metric": "auc"}, "auc": {"300": 0.81615, "450": 0.81634, "600": 0.81631}} +{"cut": 23, "tag": "xgb", "seed": 1, "n": 702680, "nf": 435, "load_s": 6, "fit_s": 195, "min": 3.6, "params": {"objective": "binary:logistic", "tree_method": "hist", "grow_policy": "lossguide", "max_leaves": 127, "max_depth": 0, "eta": 0.03, "subsample": 0.8, "colsample_bytree": 0.5, "min_child_weight": 30, "reg_lambda": 30, "max_bin": 255, "seed": 43, "eval_metric": "auc"}, "auc": {"300": 0.79425, "450": 0.79399, "600": 0.79369}} +XGB_DONE diff --git a/v12x/logs/xgb27.log b/v12x/logs/xgb27.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/v12x/mk/common.py b/v12x/mk/common.py new file mode 100644 index 0000000000000000000000000000000000000000..d0d8fa592daed326ff2d8b5a83cc2382894a3d5c --- /dev/null +++ b/v12x/mk/common.py @@ -0,0 +1,56 @@ +"""DW signals shared loader (vendored from dw_signals/common.py; paths -> work dir). +Snapshot(C): everything known from train events with ts < C (rows+labels of those events, plus click log reconstructed from their histories). +""" +import numpy as np, polars as pl, time, gc +from scipy.sparse import csr_matrix +from scipy.sparse.linalg import svds +import os +L=os.environ.get('DW_WORK','/workspace/huawei/v12/sol_work/clean25/work'); DAY=86400; J20=1737331200 # L = work dir holding train_events/train_rows/evaluation_* parquet (prep.py output) +def Jd(d): return J20+(d-20)*DAY +T0=time.time() +def log(*a): print(f'[{time.time()-T0:6.0f}s]',*a,flush=True) + +class Data: + def __init__(s): + ev=pl.read_parquet(f'{L}/train_events.parquet').sort('ts').with_row_index('e') + rows=pl.read_parquet(f'{L}/train_rows.parquet').join(ev.select('eid','e'),on='eid').sort(['e','item']) + s.NE=ev.height; assert rows.height==s.NE*10 + s.ETS=ev['ts'].to_numpy(); s.EID=ev['eid'].to_numpy() + uu,s.EU=np.unique(ev['user'].to_numpy(),return_inverse=True); s.EU=s.EU.astype(np.int32); s.NU=len(uu); s.uu=uu + s.Hlen=ev['h_items'].list.len().to_numpy().astype(np.int64) + s.OFF=np.zeros(s.NE+1,np.int64); s.OFF[1:]=np.cumsum(s.Hlen) + hi=ev['h_items'].explode().drop_nulls().to_numpy(); s.HT=ev['h_ts'].explode().drop_nulls().to_numpy().astype(np.int64) + rr=rows['item'].to_numpy() + s.items=np.unique(np.concatenate([np.unique(rr),np.unique(hi)])); s.NI=len(s.items) + s.HI=np.searchsorted(s.items,hi).astype(np.int32); s.RI=np.searchsorted(s.items,rr).astype(np.int32).reshape(s.NE,10) + s.Y=rows['clicked'].to_numpy().astype(np.int8).reshape(s.NE,10) + s.rowid=rows['rowid'].to_numpy().reshape(s.NE,10) + s.eid2e={} # not needed + s.ev=ev.select('e','eid','user','ts'); s.rows_rowid=rows.select('rowid','e','item') + del ev,rows; gc.collect() + def snap(s,X): + """click log (user,item,ts) from events with ts>20; tm=key&((1<<20)-1) + return dict(nX=nX,cu=(ui//s.NI).astype(np.int32),ci=(ui%s.NI).astype(np.int32),ct=tm*60+Jd(1)) + +def click_svd(S,D,k=32): + """item/user taste vectors from dedup click log (binary, 1/sqrt((nu+1)(ni+1)) scaling). returns item vec (V*s) L2-normed, user vec""" + u,i=S['cu'],S['ci']; key=np.unique(u.astype(np.int64)*D.NI+i); uu=(key//D.NI).astype(np.int32); ii=(key%D.NI).astype(np.int32) + nu=np.bincount(uu,minlength=D.NU).astype(np.float32); ni=np.bincount(ii,minlength=D.NI).astype(np.float32) + M=csr_matrix((1/np.sqrt((nu[uu]+1)*(ni[ii]+1)),(uu,ii)),shape=(D.NU,D.NI),dtype=np.float32) + U,sv,Vt=svds(M,k=k,random_state=0) + V=(Vt.T*sv); V/= np.linalg.norm(V,axis=1,keepdims=True)+1e-9 + return V.astype(np.float32),(U*sv).astype(np.float32) +def norm(x): return x/(np.linalg.norm(x,axis=-1,keepdims=True)+1e-9) +def event_view(a): return a.reshape(-1,10) +def within_event(x): + """z-score, demeaned, rank (0..9 ascending) within the 10-row events; x aligned with val rows""" + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + rk=m.argsort(1).argsort(1).astype(np.float32) + return ((m-mu)/sd).ravel(),(m-mu).ravel(),rk.ravel() diff --git a/v12x/mk/mkfeat.py b/v12x/mk/mkfeat.py new file mode 100644 index 0000000000000000000000000000000000000000..7164db0569dcafb25cff0b5a1b6bac6fc0cfbb5d --- /dev/null +++ b/v12x/mk/mkfeat.py @@ -0,0 +1,49 @@ +"""mkfeat.py NAME... : builds, for fold/val tables NAME (rows of v12/feat_v3/NAME.parquet), at the fold's fit cutoff X (f{k}->J20+k*DAY, val23/25 -> J20+3/5 d), + feat_prop (DW click-propensity, 28 cols), feat_tm (tm_prod0-11 + tm_prod_tail: per-dim impression-SVD taste products, k=24), + feat_tmx (interaction: row's candidate already in the event's own history x taste: tm_inh, tm_cos, tm_cos_in, tm_cos_out, tm_cos_dm, tm_inh_n). +Only events ts0)&(u0,cos,np.nan),'tm_cos_out':np.where(inh>0,np.nan,cos), + 'tm_cos_dm':(g-np.nanmean(g,1,keepdims=True)).ravel().astype(np.float32),'tm_inh_n':np.repeat(inh.reshape(-1,10).sum(1),10)} + return out,x +if __name__=='__main__': + D=Data(); log('data loaded') + for nm in sys.argv[1:]: + t=time.time(); X=CUT[nm]; V=rows_train(D,nm) + A,pn=build(D,X,V); d1=pl.DataFrame({k:A[:,i] for i,k in enumerate(pn)}).with_columns(pl.Series('rowid',V['rowid'])).select(['rowid']+pn) + o,x=tm(D,X,V); d2=pl.DataFrame(o).with_columns(pl.Series('rowid',V['rowid'])).select(['rowid']+list(o)); d3=pl.DataFrame(x).with_columns(pl.Series('rowid',V['rowid'])).select(['rowid']+list(x)) + for sub,df in (('feat_prop',d1),('feat_tm',d2),('feat_tmx',d3)): + os.makedirs(f'{O}/{sub}',exist_ok=True); df.write_parquet(f'{O}/{sub}/{nm}.parquet') + print(nm,d1.shape,d2.shape,d3.shape,round(time.time()-t),'s',flush=True) diff --git a/v12x/mk/propbuild.py b/v12x/mk/propbuild.py new file mode 100644 index 0000000000000000000000000000000000000000..e13048f8fdba928ce3edd35567d3b002684edd3b --- /dev/null +++ b/v12x/mk/propbuild.py @@ -0,0 +1,49 @@ +import numpy as np +from common import * +def build(d,C,v): + e=v['e'][::10]; nE=len(e) + ec=d.Y.max(1).astype(np.float32); ts=d.ETS; u=d.EU + nX=int(np.searchsorted(ts,C)) # events before C + # per-user aggregates over events with ts=2: g=np.diff(tss[a:bb]); gapm[uu]=g.mean(); gapsd[uu]=g.std() + ue=u[e]; tev=ts[e] + shrink=(n_cl[ue]+0.63*3)/(n_ev[ue]+3) + days=(C-J20)/DAY + col={} + col['p_nev']=n_ev[ue]; col['p_evrate']=n_ev[ue]/days; col['p_ctr']=np.where(n_ev[ue]>0,n_cl[ue]/np.maximum(n_ev[ue],1),np.nan); col['p_ctr_s']=shrink + col['p_l3']=l3[ue]; col['p_streak']=streak[ue]; col['p_gapm']=gapm[ue]/3600; col['p_gapsd']=gapsd[ue]/3600 + col['p_since_ev']=np.where(np.isfinite(last[ue]),(tev-last[ue])/3600,np.nan); col['p_since_first']=np.where(np.isfinite(first[ue]),(tev-first[ue])/3600,np.nan) + col['p_gap_ratio']=col['p_since_ev']/(col['p_gapm']+1) + # own history + hl=d.Hlen[e]; col['h_len']=hl.astype(np.float32) + hmin=np.full(nE,np.nan); hmax=np.full(nE,np.nan); hgm=np.full(nE,np.nan); hgsd=np.full(nE,np.nan); hglast=np.full(nE,np.nan); h1=np.zeros(nE); h3=np.zeros(nE); h7=np.zeros(nE); hwarm=np.zeros(nE); hwin=np.zeros(nE); huniq=np.zeros(nE) + for i,ee in enumerate(e): + a,bb=d.OFF[ee],d.OFF[ee+1] + if bb-a==0: continue + t=d.HT[a:bb]; it=d.HI[a:bb]; tt=tev[i] + hmin[i]=(tt-t[0])/3600; hmax[i]=(tt-t[-1])/3600; huniq[i]=len(set(it.tolist())) + h1[i]=(t>tt-DAY).sum(); h3[i]=(t>tt-3*DAY).sum(); h7[i]=(t>tt-7*DAY).sum(); hwarm[i]=(t=J20).sum() + if bb-a>=2: g=np.diff(t); hgm[i]=g.mean()/3600; hgsd[i]=g.std()/3600; hglast[i]=g[-1]/3600 + col.update(h_span=hmin,h_since_last=hmax,h_gap_mean=hgm,h_gap_sd=hgsd,h_gap_last=hglast,h_1d=h1,h_3d=h3,h_7d=h7,h_warm=hwarm,h_win=hwin,h_uniq=huniq,h_uniq_ratio=huniq/np.maximum(hl,1)) + col['h_rate_day']=np.where(hl>=2,(hl-1)/np.maximum(hmin-hmax,1e-3)*24,np.nan) + # clicks per event: window clicks (in train, ts0,n_ev[ue]/np.maximum((C-J20)/3600/24,1e-3),np.nan) # events per day + col['implied_cpe']=col['h_rate_day']/(erate+0.5) + col['gap_over_hgap']=col['p_gapm']/(col['h_gap_mean']+1) + X=np.column_stack([np.repeat(np.asarray(col[k],np.float32),10) for k in col]).astype(np.float32) + return X,list(col) + diff --git a/v12x/run_q2.sh b/v12x/run_q2.sh new file mode 100644 index 0000000000000000000000000000000000000000..330ab0e2609ea30065bc284d6f9b414658d7c3d1 --- /dev/null +++ b/v12x/run_q2.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# queue after XGB: lgb sanity, lambdarank, xendcg, max_bin. one heavy job at a time. +while pgrep -f run_xgb.sh >/dev/null; do sleep 20; done +cd /workspace/huawei/v12; export POLARS_MAX_THREADS=8 OMP_NUM_THREADS=8 PYTHONUNBUFFERED=1 +run(){ t0=$(date +%s); /workspace/venv/bin/python /workspace/huawei/v12x/xlgb.py "$@" 2>&1 | grep -v Warn | tail -2 + echo "{\"args\":\"$*\",\"wall_s\":$(( $(date +%s)-t0 ))}" >> /workspace/huawei/v12x/ens/timings.jsonl; } +LR='{"objective":"lambdarank","lambdarank_truncation_level":10,"metric":"None"}' +XE='{"objective":"rank_xendcg","metric":"None"}' +run 25 lgb255 --rounds 1000 --save 800,1000 +run 25 lrank --group --rounds 1000 --save 500,1000 --params "$LR" +run 23 lrank --group --rounds 1000 --save 500,1000 --params "$LR" +run 25 xendcg --group --rounds 1000 --save 500,1000 --params "$XE" +run 23 xendcg --group --rounds 1000 --save 500,1000 --params "$XE" +run 25 mb1023 --rounds 1000 --save 1000 --params '{"max_bin":1023}' +run 23 mb1023 --rounds 1000 --save 1000 --params '{"max_bin":1023}' +run 25 mb511 --rounds 1000 --save 1000 --params '{"max_bin":511}' +run 23 mb511 --rounds 1000 --save 1000 --params '{"max_bin":511}' +echo Q2_DONE diff --git a/v12x/run_q3.sh b/v12x/run_q3.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0065f998863f26a57e7892826208e39dfe11f31 --- /dev/null +++ b/v12x/run_q3.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# feature retrains (dwbig recipe) after q2 +while pgrep -f "run_q2.sh|run_xgb.sh" >/dev/null; do sleep 20; done +cd /workspace/huawei/v12; export POLARS_MAX_THREADS=8 OMP_NUM_THREADS=8 PYTHONUNBUFFERED=1 +B=feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw +run(){ t0=$(date +%s); /workspace/venv/bin/python /workspace/huawei/v12x/xlgb.py "$@" 2>&1 | grep -v Warn | tail -2 + echo "{\"args\":\"$*\",\"wall_s\":$(( $(date +%s)-t0 ))}" >> /workspace/huawei/v12x/ens/timings.jsonl; } +for c in 25 23; do + run $c f_prtm --lr $B,feat_prop,feat_tm --rounds 1000 --save 1000 + run $c f_tm --lr $B,feat_tm --rounds 1000 --save 1000 + run $c f_pr --lr $B,feat_prop --rounds 1000 --save 1000 + run $c f_tmx --lr $B,feat_tm,feat_tmx --rounds 1000 --save 1000 + run $c f_all --lr $B,feat_prop,feat_tm,feat_tmx --rounds 1000 --save 1000 +done +echo Q3_DONE diff --git a/v12x/run_xgb.sh b/v12x/run_xgb.sh new file mode 100644 index 0000000000000000000000000000000000000000..afc868c4b1090d4b084c9752d66b4d21b7952cb7 --- /dev/null +++ b/v12x/run_xgb.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /workspace/huawei/v12; export POLARS_MAX_THREADS=8 OMP_NUM_THREADS=8 PYTHONUNBUFFERED=1 +for sc in "25 0" "23 0" "25 1" "23 1"; do set -- $sc + t0=$(date +%s); /workspace/venv/bin/python /workspace/huawei/v12x/xens.py $1 xgb --seed $2 --rounds 600 --save 300,450,600 2>&1 | grep -v "^\s*\[bt\]" | tail -2 + echo "{\"tag\":\"xgb\",\"cut\":$1,\"seed\":$2,\"wall_s\":$(( $(date +%s)-t0 ))}" >> /workspace/huawei/v12x/ens/timings.jsonl +done; echo XGB_DONE diff --git a/v12x/xens.py b/v12x/xens.py new file mode 100644 index 0000000000000000000000000000000000000000..9772781829e215e514fa17325ad74d2cbf15121d --- /dev/null +++ b/v12x/xens.py @@ -0,0 +1,39 @@ +"""XGBoost (hist, CPU) on the exact v12 feature matrix/folds used by dwbig. Run with cwd=/workspace/huawei/v12 (uses its cl.matrix + feat dirs). +python xens.py CUT TAG [--pick] [--rounds N] [--save r1,r2] [--seed 0] [--params json] +--pick : train on all training folds except the LAST one, early-stop on the last training fold (never the val cut); prints best iteration, no val predictions. +else : train on all training folds (same as dwbig: f(max(1,cut-24))..f(cut-21), all negatives, drop sq_(pdot|pcos|mdot|mcos)), save raw margins at --save rounds to v12x/ens/{tag}_c{CUT}_s{seed}.npy""" +import sys, os, re, json, gc, time, argparse +sys.path.insert(0,'/workspace/huawei/v12x/pylib'); sys.path.insert(0,'/workspace/huawei/v12x') +import numpy as np, polars as pl, xgboost as xgb +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--pick',action='store_true') +ap.add_argument('--rounds',type=int,default=1500); ap.add_argument('--save',default='500'); ap.add_argument('--seed',type=int,default=0) +ap.add_argument('--params',default='{}'); ap.add_argument('--threads',type=int,default=8); ap.add_argument('--lr',default='feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw') +a=ap.parse_args(); t0=time.time(); DROP='^sq_(pdot|pcos|mdot|mcos)$' +extra=tuple(x for x in a.lr.split(',') if x) +trn=[f'f{k}' for k in range(max(1,a.cut-24),a.cut-20)] +def load(names): # +-inf -> +-1e15 (xgboost rejects inf); NaN kept as missing + Xs=[];ys=[];fl=None + for n in names: + X,m,f=matrix(n,1.0,variant='notime',extra=extra); keep=[i for i,c in enumerate(f) if not re.search(DROP,c)]; fl=[f[i] for i in keep] + Xs.append(np.clip(np.ascontiguousarray(X[:,keep]),-1e15,1e15)); ys.append(m['clicked'].to_numpy()); del X,m; gc.collect() + return np.concatenate(Xs),np.concatenate(ys),fl +P=dict(objective='binary:logistic',tree_method='hist',grow_policy='lossguide',max_leaves=127,max_depth=0,eta=0.03,subsample=0.8,colsample_bytree=0.5, + min_child_weight=30,reg_lambda=30,max_bin=255,nthread=a.threads,seed=42+a.seed,eval_metric='auc'); P.update(json.loads(a.params)) +if a.pick: + X,y,fl=load(trn[:-1]); Xh,yh,_=load(trn[-1:]) + dtr=xgb.QuantileDMatrix(X,y,feature_names=fl,max_bin=P['max_bin']); del X; dh=xgb.DMatrix(Xh,yh,feature_names=fl); del Xh; gc.collect() + ev={}; b=xgb.train(P,dtr,a.rounds,evals=[(dh,'hold')],early_stopping_rounds=100,evals_result=ev,verbose_eval=50) + r=dict(pick=True,cut=a.cut,train=trn[:-1],hold=trn[-1],best_iter=b.best_iteration,best_auc=b.best_score,fit_s=round(time.time()-t0)); print(json.dumps(r)); open('/workspace/huawei/v12x/ens/xlog.jsonl','a').write(json.dumps(r)+'\n'); sys.exit() +X,y,fl=load(trn); t_load=time.time()-t0 +dtr=xgb.QuantileDMatrix(X,y,feature_names=fl,max_bin=P['max_bin']); del X; gc.collect() +vn='eval' if a.cut==27 else f'val{a.cut}' +Xv,mv,fv=matrix(vn,1.0,variant='notime',extra=extra); Xv=np.clip(Xv[:,[fv.index(c) for c in fl]],-1e15,1e15); dv=xgb.DMatrix(Xv,feature_names=fl) +RS=[int(r) for r in a.save.split(',')]; assert max(RS)<=a.rounds +t1=time.time(); b=xgb.train({k:v for k,v in P.items() if k!='eval_metric'},dtr,max(RS)); t_fit=time.time()-t1 +S=np.stack([b.predict(dv,iteration_range=(0,r),output_margin=True) for r in RS],1) +np.save(f'/workspace/huawei/v12x/ens/{a.tag}_c{a.cut}_s{a.seed}.npy',S.astype(np.float32)) +msg=dict(cut=a.cut,tag=a.tag,seed=a.seed,n=len(y),nf=len(fl),load_s=round(t_load),fit_s=round(t_fit),min=round((time.time()-t0)/60,1),params={k:P[k] for k in P if k not in('nthread',)}) +if a.cut!=27: msg['auc']={r:round(roc_auc_score(mv['clicked'].to_numpy(),S[:,i]),5) for i,r in enumerate(RS)} +print(json.dumps(msg),flush=True); open('/workspace/huawei/v12x/ens/xlog.jsonl','a').write(json.dumps(msg)+'\n') diff --git a/v12x/xgb_an.py b/v12x/xgb_an.py new file mode 100644 index 0000000000000000000000000000000000000000..d41ec1d309916058f5e999ab5414c97477ae96ba --- /dev/null +++ b/v12x/xgb_an.py @@ -0,0 +1,13 @@ +from analyze import * +for c in (25,23): + big,sm,ml=base(c); z0=(big+sm+2*ml)/4 + print('cut',c,'v12 blend',auc(c,z0),'dwbig s0',auc(c,big)) + try: + b1=ld(f'{V}/dwbig_c{c}_s1.npy',1); print(' dwbig s1',auc(c,b1)) + except Exception as e: print(e) + for s in (0,1): + a=np.load(f'{X}/xgb_c{c}_s{s}.npy'); print(' xgb s%d shape'%s,a.shape,[auc(c,a[:,i]) for i in range(a.shape[1])]) + for ci in range(a.shape[1]): + x=a[:,ci] + res={w:auc(c,(big+sm+2*ml+w*x)/(4+w)) for w in (.5,1,2)} + print(' col',ci,'blend',res) diff --git a/v12x/xlgb.py b/v12x/xlgb.py new file mode 100644 index 0000000000000000000000000000000000000000..cc027d736efff31cb01fe9bfaaf64a535e8bdefc --- /dev/null +++ b/v12x/xlgb.py @@ -0,0 +1,31 @@ +"""LightGBM variants on the v12 feature matrix (dwbig-like). python xlgb.py CUT TAG [--seed 0] [--rounds 1000] [--save 1000] [--params json] [--group] +Same loader/folds/drop as v12 ens.py (cwd=/workspace/huawei/v12, cl from v12x). --group: ranking objectives, group=[10]*n_events (rows are event-major, verified). +Saves v12x/ens/{tag}_c{CUT}_s{seed}.npy (raw scores at --save rounds).""" +import sys, os, re, json, gc, time, argparse +sys.path.insert(0,'/workspace/huawei/v12x') +import numpy as np, lightgbm as lgb +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--seed',type=int,default=0) +ap.add_argument('--rounds',type=int,default=1000); ap.add_argument('--save',default='1000'); ap.add_argument('--params',default='{}'); ap.add_argument('--group',action='store_true') +ap.add_argument('--threads',type=int,default=8); ap.add_argument('--lr',default='feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw') +a=ap.parse_args(); t0=time.time(); DROP='^sq_(pdot|pcos|mdot|mcos)$'; extra=tuple(x for x in a.lr.split(',') if x) +trn=[f'f{k}' for k in range(max(1,a.cut-24),a.cut-20)] +Xs=[];ys=[] +for n in trn: + X,m,f=matrix(n,1.0,variant='notime',extra=extra); keep=[i for i,c in enumerate(f) if not re.search(DROP,c)]; fl=[f[i] for i in keep] + Xs.append(np.ascontiguousarray(X[:,keep])); ys.append(m['clicked'].to_numpy()); del X,m; gc.collect() +X=np.concatenate(Xs); del Xs; y=np.concatenate(ys); gc.collect(); assert len(y)%10==0 +P=dict(objective='binary',learning_rate=0.015,num_leaves=127,min_data_in_leaf=200,feature_fraction=0.5,bagging_fraction=0.8,bagging_freq=1,lambda_l2=30.0, + max_bin=255,force_col_wise=True,verbose=-1,num_threads=a.threads,seed=42+a.seed); P.update(json.loads(a.params)) +t_load=time.time()-t0; t1=time.time() +ds=lgb.Dataset(X,y,feature_name=fl,free_raw_data=True,group=[10]*(len(y)//10) if a.group else None,params={k:P[k] for k in ('max_bin','force_col_wise','max_bin_by_feature') if k in P}).construct(); t_ds=time.time()-t1; del X; gc.collect() +vn='eval' if a.cut==27 else f'val{a.cut}' +Xv,mv,fv=matrix(vn,1.0,variant='notime',extra=extra); Xv=Xv[:,[fv.index(c) for c in fl]] +RS=[int(r) for r in a.save.split(',')]; t1=time.time(); mdl=lgb.train(P,ds,max(RS)); t_fit=time.time()-t1 +S=np.stack([mdl.predict(Xv,num_iteration=r,raw_score=True) for r in RS],1) +np.save(f'/workspace/huawei/v12x/ens/{a.tag}_c{a.cut}_s{a.seed}.npy',S.astype(np.float32)) +imp=dict(zip(fl,mdl.feature_importance('gain').tolist())); json.dump(imp,open(f'/workspace/huawei/v12x/ens/{a.tag}_c{a.cut}_s{a.seed}_gain.json','w')) +msg=dict(cut=a.cut,tag=a.tag,seed=a.seed,n=len(y),nf=len(fl),load_s=round(t_load),ds_s=round(t_ds),fit_s=round(t_fit),min=round((time.time()-t0)/60,1),obj=P['objective'],max_bin=P['max_bin']) +if a.cut!=27: msg['auc']={r:round(roc_auc_score(mv['clicked'].to_numpy(),S[:,i]),5) for i,r in enumerate(RS)} +print(json.dumps(msg),flush=True); open('/workspace/huawei/v12x/ens/xlog.jsonl','a').write(json.dumps(msg)+'\n') diff --git a/v13/NOTES.md b/v13/NOTES.md new file mode 100644 index 0000000000000000000000000000000000000000..b1b66185706d38071e3ea6535f13f29c2f0f8de6 --- /dev/null +++ b/v13/NOTES.md @@ -0,0 +1,17 @@ +# v13 NOTES (started 2026-10-28 00:17 Dublin) -- base v12 method b: C25 .81723, C23 .79566, C26 .82604 +## step 0: cleanup (Eric standing yes, pt108u) +Freed ~17 GB (df used 75G -> 59G): removed v8/cache_mx (7.7G, v8's own cache; v9d/v8b cache_mx used by v12 untouched, verified matrix(eval/val25/f3) loads), + v2work/st_fit.csv (3.9G), v4/feat (1.9G), v3/feat (0.9G), parallel/lag + parallel_v6/lag (0.85G), v10 feat_sq10_{bs*,ep2*,p,q,e,c} variants (~1G; feat_sq10 kept, v11 links it), + all __pycache__ in v3/v4/v8/v8b/v9/v9d/v10/v11/v12/tools, /tmp leftovers (chrome temp, scoped_dir*, my pngs/py/json), pip cache. + KEPT: v8/train.csv+evaluation.csv (-> v2work CSVs, v8 pipeline inputs), v2work feat dirs (v8/feat_rc symlinks), v8b cache_mx, v11 msA/B/C + feat_sqC, v12 feat_dw, dw_signals, ens, out/, v5, parts/zips. +## setup +v13 = copy of v12 code (paths patched), symlinks to v11 feat dirs + v12/feat_dw; ens/ symlinks v12/v11 components. Extra feature dirs go in v13/feat_ev etc. +## note (00:19 Dublin, from parent): Eric uploaded v12 -> public LB 0.817935 (v8 was 0.8144, +0.0035); local C25 gain vs v8 was ~+0.004 => C25 is a reliable guide. +## step 1: headroom experiment exp1.py (event-level any-click model on v12 logits, OOF) -- running (first launch interrupted) + +## step 1 results (exp1/exp1b/exp2) +- post-hoc event offset (OOF on val rows, init e0): event AUC .67288->.6747/.6765 but global C25 -.0013 (a=.5) .. -.005 (a=1); a=.1 ~0 (±.0001); C23 same. => no headroom as post-hoc offset. +- exp2 (row-level residual boost on v12 logits, OOF on val25 rows, 5-fold by user, event aggregates of own slate, top-80 feats): base +.00003, +evconst+agg +.00029, +rowdev +.00033, +all +.00041 vs v12 .81723. Small, but val-only training (35k events). Next: put event aggs/rowdev into row models trained on train folds (feat_ev). +- feat_ev built (mkev.py, K=80 top feats by f1 within-event corr; agg mean/max/min/std + row dev = 400 cols; all 10 tables). Testing ev_s (dw_s + feat_ev) c25,c23 via b1.sh -> ens/ev_s_* +- RESULT ev_s (dw_s + feat_ev, event aggs in row model, train folds): c25 .81593 vs v12 dw_s .81595; c23 .79418 vs .79415 -> no gain (sum of both = seed averaging only). Priority 1 ruled out as features too. +- RESULT aw_s (fold recency weight agew .7): c25 .81589 (500)/.81601(400) vs .81595; c23 .7938 vs .79415 -> no gain. FINAL: nothing beats v12; v13 NOT shipped. Packaging task should use v12. diff --git a/v13/b1.sh b/v13/b1.sh new file mode 100644 index 0000000000000000000000000000000000000000..fd3de2032d45595028a52002eef6f8f1c561700c --- /dev/null +++ b/v13/b1.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cd /workspace/huawei/v13 +LP='"max_bin":63,"force_col_wise":true'; SLOW="\"learning_rate\":0.03,\"feature_fraction\":0.55,\"num_leaves\":95,\"lambda_l2\":30,$LP" +BIG='{"learning_rate":0.015,"num_leaves":127,"feature_fraction":0.5,"lambda_l2":30,"min_data_in_leaf":200,"max_bin":255,"force_col_wise":true}' +L=feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw,feat_ev +for c in ${CUTS:-25 23}; do ./e.sh ev_s "$c" 500 400,500 0.5 "{$SLOW}" $L; done +echo B1_DONE diff --git a/v13/b3.sh b/v13/b3.sh new file mode 100644 index 0000000000000000000000000000000000000000..b1a4a83b5cdeb52e5486daf106bb44dfab7237c3 --- /dev/null +++ b/v13/b3.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /workspace/huawei/v13 +LP='"max_bin":63,"force_col_wise":true'; SLOW="\"learning_rate\":0.03,\"feature_fraction\":0.55,\"num_leaves\":95,\"lambda_l2\":30,$LP" +L=feat_lr2,feat_tt,feat_pp,feat_sqC,feat_dw +for c in 25 23; do ./e.sh aw_s "$c" 500 400,500 0.5 "{$SLOW}" $L --agew 0.7; done +echo B3_DONE diff --git a/v13/cl.py b/v13/cl.py new file mode 100644 index 0000000000000000000000000000000000000000..45524530c09f7f78e4cc444c08603e7fae3f15e2 --- /dev/null +++ b/v13/cl.py @@ -0,0 +1,65 @@ +"""Shared loader + feature-matrix cache (step C). Matrices cached as parquet keyed by hash(table names, feature list, sample).""" +import os, sys, hashlib, json, gc, numpy as np, polars as pl +sys.path.insert(0,'/workspace/huawei/v9'); sys.path.insert(1,'/workspace/huawei/v3') +from feats import DROP +from features import feature_cols, GROUPS +V9='/workspace/huawei/v9'; D='/workspace/huawei/v13'; P2=f'{V9}/feat'; P3=f'{V9}/feat_v3'; CACHE='/workspace/huawei/v9d/val/cache_mx' # read-only base cache +V2_OF={'val23':'val23','f1':'h1','f2':'h2','f3':'h3','f4':'h4','f5':'h5','f6':'val26','val25':'val25','val26':'val26','eval':'eval'} +V2_OF.update({f'{p}{k}':f'{p}{k}' for p in 'eEd' for k in range(1,7)}) +TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour'] +KEEP=['rowid','eid','u','ts','clicked','us_cold','us_nohist','tm_stale'] +def load_tab(n, extra=(), variant='notime'): + a=pl.read_parquet(f'{P3}/{n}.parquet'); b=pl.read_parquet(f'{P2}/{V2_OF[n]}.parquet') + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.read_parquet(f'{D}/{sub}/{n}.parquet'),on='rowid',how='left') + if '+' in variant: + import mods; a=mods.apply(a,n,variant) + return a +def featlist(df, variant='notime'): + f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith(('v2_','x_','r_','s_','ev2_','rc_','sh_','hs_','lr','tA_','tB_','uw_','la_','sq_'))] + if 'notime' in variant: f=[c for c in f if c not in TIME] + if variant.split('+')[0]=='stale': f=[c for c in f if c not in TIME or c=='tm_stale'] + return f +def usample(df, frac): + if frac>=1: return df + return df.filter((pl.col('u').hash(seed=7) % 10000) < int(frac*10000)) +FL=[None] +def load_tab_rows(n, rid, extra=(), variant='notime'): + """same as load_tab but restricted to rowids rid (polars Series) -> low memory""" + a=pl.scan_parquet(f'{P3}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + b=pl.scan_parquet(f'{P2}/{V2_OF[n]}.parquet').filter(pl.col('rowid').is_in(rid)).collect() + f2=[c for c in b.columns if c not in DROP] + a=a.join(b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left') + for sub in (os.environ.get('XDIR','feat_x'),)+(('feat_x2',) if os.path.isdir(f'{D}/feat_x2') else ())+tuple(extra): + a=a.join(pl.scan_parquet(f'{D}/{sub}/{n}.parquet').filter(pl.col('rowid').is_in(rid)).collect(),on='rowid',how='left') + return a +def build_chunked(n, extra, variant, frac, nch=4): + rid_all=pl.read_parquet(f'{P3}/{n}.parquet',columns=['rowid'])['rowid']; parts=[] + step=(len(rid_all)+nch-1)//nch + for i in range(nch): + rid=rid_all.slice(i*step,step); t=usample(load_tab_rows(n,rid,extra,variant),frac) + # restore original order within chunk + t=pl.DataFrame({'rowid':rid}).join(t,on='rowid',how='inner',maintain_order='left') + fl=featlist(t,variant); FL[0]=fl + t=t.select(list(dict.fromkeys([c for c in KEEP if c in t.columns]+fl))) + t=t.with_columns([pl.col(c).cast(pl.Float32) for c in fl if t[c].dtype!=pl.Float32 and c not in KEEP]) + parts.append(t); gc.collect() + return pl.concat(parts) +BASE=('feat_lr2','feat_tt','feat_pp') +def matrix(n, frac=1.0, variant='notime', extra=()): + """v9: base = cached v8-lite matrix (read-only, from v9d/val/cache_mx); new dirs (feat_tm*) joined on rowid. + returns (X float32, meta DataFrame, feats).""" + base=tuple(e for e in extra if e in BASE); new=[e for e in extra if e not in BASE]; assert base==BASE, extra + key=hashlib.md5(json.dumps([n,frac,variant,list(base),os.path.getmtime(f'{P3}/{n}.parquet')]).encode()).hexdigest()[:12] + fp=f'{CACHE}/{n}_{key}.parquet'; assert os.path.exists(fp), fp + fl=json.load(open(fp+'.json')); t=pl.read_parquet(fp) + for sub in new: + e=pl.read_parquet(f'{D}/{sub}/{n}.parquet'); nc=[c for c in e.columns if c!='rowid' and c not in fl] + t=t.join(e.select(['rowid']+nc),on='rowid',how='left',maintain_order='left'); fl=fl+nc + drop=os.environ.get('V9DROP') + if drop: + import re; fl=[c for c in fl if not re.search(drop,c)] + X=t.select(fl).cast(pl.Float32).to_numpy(); meta=t.select([c for c in KEEP if c in t.columns]); del t; gc.collect() + return X, meta, fl diff --git a/v13/cmp10.py b/v13/cmp10.py new file mode 100644 index 0000000000000000000000000000000000000000..9c49191f123353c5c4b657fe6052a2029f6e7a68 --- /dev/null +++ b/v13/cmp10.py @@ -0,0 +1,32 @@ +"""cmp10.py TAG [TAG...] [--w 0.2] [--sq DIR] [--ref trim] [--refw 0.2]: z=(b400+s500)/2, zf=(1-w)z+w*sq_logit (sq from DIR/val{c}.parquet, +default v8 feat_sqv/all500); AUC overall, by staleness bucket, staleness-reweighted (66.6/16.6/16.8). ref = v8 trim at w 0.2 with all500 sq.""" +import sys, json, numpy as np, polars as pl +from sklearn.metrics import roc_auc_score as A +V8='/workspace/huawei/v8'; EV=np.array([0.6662,0.1655,0.1682]); av=sys.argv[1:] +def opt(k,d): + if k in av: i=av.index(k); v=av[i+1]; del av[i:i+2]; return v + return d +w=float(opt('--w','0.2')); sqd=opt('--sq',f'{V8}/feat_sqv/all500'); ref=opt('--ref','trim'); refw=float(opt('--refw','0.2')); tags=av +Y={};BK={};W={};SQ={};SQ8={} +for c in (25,26,23): + v=pl.read_parquet(f'/workspace/huawei/v10/feat_v3/val{c}.parquet',columns=['rowid','clicked']) + Y[c]=v['clicked'].to_numpy() + for D_,T_ in ((sqd,SQ),(f'{V8}/feat_sqv/all500',SQ8)): + T_[c]=v.join(pl.read_parquet(f'{D_}/val{c}.parquet',columns=['rowid','sq_logit']),on='rowid',how='left',maintain_order='left')['sq_logit'].to_numpy() + st=pl.read_parquet(f'/workspace/huawei/v10/feat_tmB2/val{c}.parquet')['tmR_stale_h'].to_numpy(); b=np.digitize(st,[24,48]); BK[c]=b + sh=np.bincount(b,minlength=3)/len(b); pres=sh>0; tgt=np.where(pres,EV,0); tgt/=tgt.sum(); W[c]=(tgt/np.where(pres,sh,1))[b] +def z(tag,c,w,sq): + b=np.load(f'/workspace/huawei/v10/ens/{tag}_b_c{c}_s0.npy')[:,1]; s=np.load(f'/workspace/huawei/v10/ens/{tag}_s_c{c}_s0.npy')[:,2]; return (1-w)*(b+s)/2+w*sq[c] +def stats(z,c): + y=Y[c]; b=BK[c]; r=dict(auc=A(y,z),rw=A(y,z,sample_weight=W[c])) + for k,nm in enumerate(('b0_24','b24_48','b48p')): + m=b==k; r[nm]=A(y[m],z[m]) if m.sum()>1000 else np.nan + return r +if __name__=='__main__': + for tag in tags: + for c in (25,23,26): + try: zz=z(tag,c,w,SQ) + except FileNotFoundError: continue + s=stats(zz,c); s0=stats(z(ref,c,refw,SQ8),c) + r=dict(tag=tag,w=w,sq=sqd.split('/')[-1],cut=c,**{k:round(v,5) for k,v in s.items()},**{'d_'+k:round(s[k]-s0[k],5) for k in s}) + print(json.dumps(r),flush=True); open('cmp10_results.jsonl','a').write(json.dumps(r)+'\n') diff --git a/v13/cmp11.py b/v13/cmp11.py new file mode 100644 index 0000000000000000000000000000000000000000..9fea091571b303b1eb9fcef53f1fbc79c8a74949 --- /dev/null +++ b/v13/cmp11.py @@ -0,0 +1,27 @@ +"""cmp11.py NAME SPEC [--sqw w --sq DIR]: SPEC = items joined by '+', item = tag:col[:seeds] (seeds comma list, default 0) -> mean raw score. +Reports AUC/rw/buckets per cut (25,23,26) and deltas vs v8 trim (w .2 sq8) and v10B. Appends to cmp11_results.jsonl.""" +import sys, json, numpy as np +sys.argv=[sys.argv[0]]+sys.argv[1:] +import cmp10 as C +from sklearn.metrics import roc_auc_score as A +av=sys.argv[1:] +def opt(k,d): + if k in av: i=av.index(k); v=av[i+1]; del av[i:i+2]; return v + return d +sqw=float(opt('--sqw','0')); name,spec=av[0],av[1] +def load(item,c): + w=1.0 + if '*' in item: w,item=item.split('*'); w=float(w) + p=item.split(':'); tag,col=p[0],int(p[1]); seeds=p[2].split(',') if len(p)>2 else ['0'] + return w*np.mean([np.load(f'ens/{tag}_c{c}_s{s}.npy')[:,col] for s in seeds],0) +def zspec(spec,c): + it=spec.split('+'); W=sum(float(i.split('*')[0]) if '*' in i else 1.0 for i in it); return np.sum([load(i,c) for i in it],0)/W +def ref10(c): + b=np.load(f'/workspace/huawei/v10/ens/v10B_b_c{c}_s0.npy')[:,1]; s=np.load(f'/workspace/huawei/v10/ens/v10B_s_c{c}_s0.npy')[:,2]; return (b+s)/2 +for c in (25,23,26): + try: z=zspec(spec,c) + except FileNotFoundError: continue + if sqw: z=(1-sqw)*z+sqw*C.SQ[c] + s=C.stats(z,c); s8=C.stats(C.z('trim',c,0.2,C.SQ8) if False else (0.8*(np.load(f'/workspace/huawei/v10/ens/trim_b_c{c}_s0.npy')[:,1]+np.load(f'/workspace/huawei/v10/ens/trim_s_c{c}_s0.npy')[:,2])/2+0.2*C.SQ8[c]),c); s10=C.stats(ref10(c),c) + r=dict(name=name,spec=spec,cut=c,**{k:round(v,5) for k,v in s.items()},**{'d8_'+k:round(s[k]-s8[k],5) for k in ('auc','rw')},**{'d10_'+k:round(s[k]-s10[k],5) for k in s}) + print(json.dumps(r),flush=True); open('cmp11_results.jsonl','a').write(json.dumps(r)+'\n') diff --git a/v13/cmp11_results.jsonl b/v13/cmp11_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..919d04f8c3331e4dbc8e1eb73ab70e6722705554 --- /dev/null +++ b/v13/cmp11_results.jsonl @@ -0,0 +1,24 @@ +{"name": "v11chk", "spec": "sCbig:1+sC_s:1+2*mlp2:3:0,1,2", "cut": 25, "auc": 0.81524, "rw": 0.81523, "b0_24": 0.81868, "b24_48": 0.80085, "b48p": NaN, "d8_auc": 0.00202, "d8_rw": 0.00202, "d10_auc": 0.00135, "d10_rw": 0.00135, "d10_b0_24": 0.00129, "d10_b24_48": 0.00163, "d10_b48p": NaN} +{"name": "v11chk", "spec": "sCbig:1+sC_s:1+2*mlp2:3:0,1,2", "cut": 23, "auc": 0.79424, "rw": 0.79308, "b0_24": 0.80342, "b24_48": 0.77518, "b48p": 0.76702, "d8_auc": 0.00369, "d8_rw": 0.00368, "d10_auc": 0.00323, "d10_rw": 0.00324, "d10_b0_24": 0.00304, "d10_b24_48": 0.00375, "d10_b48p": 0.00346} +{"name": "b_full", "spec": "dwbig:1+dw_s:1+2*dwmlp:3:0,1,2", "cut": 25, "auc": 0.81723, "rw": 0.81722, "b0_24": 0.82067, "b24_48": 0.80284, "b48p": NaN, "d8_auc": 0.00401, "d8_rw": 0.00401, "d10_auc": 0.00334, "d10_rw": 0.00334, "d10_b0_24": 0.00328, "d10_b24_48": 0.00362, "d10_b48p": NaN} +{"name": "b_full", "spec": "dwbig:1+dw_s:1+2*dwmlp:3:0,1,2", "cut": 23, "auc": 0.79566, "rw": 0.79449, "b0_24": 0.80475, "b24_48": 0.77704, "b48p": 0.7684, "d8_auc": 0.0051, "d8_rw": 0.00509, "d10_auc": 0.00464, "d10_rw": 0.00465, "d10_b0_24": 0.00437, "d10_b24_48": 0.00562, "d10_b48p": 0.00484} +{"name": "b_lgbonly", "spec": "dwbig:1+dw_s:1+2*mlp2:3:0,1,2", "cut": 25, "auc": 0.81668, "rw": 0.81668, "b0_24": 0.82011, "b24_48": 0.8024, "b48p": NaN, "d8_auc": 0.00347, "d8_rw": 0.00347, "d10_auc": 0.0028, "d10_rw": 0.0028, "d10_b0_24": 0.00271, "d10_b24_48": 0.00318, "d10_b48p": NaN} +{"name": "b_lgbonly", "spec": "dwbig:1+dw_s:1+2*mlp2:3:0,1,2", "cut": 23, "auc": 0.79523, "rw": 0.79407, "b0_24": 0.80432, "b24_48": 0.77648, "b48p": 0.76816, "d8_auc": 0.00467, "d8_rw": 0.00467, "d10_auc": 0.00421, "d10_rw": 0.00423, "d10_b0_24": 0.00394, "d10_b24_48": 0.00506, "d10_b48p": 0.0046} +{"name": "v11chk_s0", "spec": "sCbig:1+sC_s:1+2*mlp2:3", "cut": 25, "auc": 0.81522, "rw": 0.81521, "b0_24": 0.81867, "b24_48": 0.80081, "b48p": NaN, "d8_auc": 0.00201, "d8_rw": 0.00201, "d10_auc": 0.00134, "d10_rw": 0.00134, "d10_b0_24": 0.00128, "d10_b24_48": 0.00158, "d10_b48p": NaN} +{"name": "v11chk_s0", "spec": "sCbig:1+sC_s:1+2*mlp2:3", "cut": 23, "auc": 0.79417, "rw": 0.793, "b0_24": 0.80336, "b24_48": 0.77505, "b48p": 0.76699, "d8_auc": 0.00361, "d8_rw": 0.0036, "d10_auc": 0.00315, "d10_rw": 0.00317, "d10_b0_24": 0.00298, "d10_b24_48": 0.00363, "d10_b48p": 0.00343} +{"name": "v11chk_s0", "spec": "sCbig:1+sC_s:1+2*mlp2:3", "cut": 26, "auc": 0.82434, "rw": 0.82434, "b0_24": 0.82434, "b24_48": NaN, "b48p": NaN, "d8_auc": 0.00245, "d8_rw": 0.00245, "d10_auc": 0.00128, "d10_rw": 0.00128, "d10_b0_24": 0.00128, "d10_b24_48": NaN, "d10_b48p": NaN} +{"name": "b_full_s0", "spec": "dwbig:1+dw_s:1+2*dwmlp:3", "cut": 25, "auc": 0.8172, "rw": 0.8172, "b0_24": 0.82064, "b24_48": 0.80286, "b48p": NaN, "d8_auc": 0.00399, "d8_rw": 0.00399, "d10_auc": 0.00332, "d10_rw": 0.00332, "d10_b0_24": 0.00325, "d10_b24_48": 0.00364, "d10_b48p": NaN} +{"name": "b_full_s0", "spec": "dwbig:1+dw_s:1+2*dwmlp:3", "cut": 23, "auc": 0.79554, "rw": 0.79437, "b0_24": 0.80463, "b24_48": 0.77688, "b48p": 0.76827, "d8_auc": 0.00498, "d8_rw": 0.00497, "d10_auc": 0.00452, "d10_rw": 0.00453, "d10_b0_24": 0.00425, "d10_b24_48": 0.00546, "d10_b48p": 0.00471} +{"name": "b_full_s0", "spec": "dwbig:1+dw_s:1+2*dwmlp:3", "cut": 26, "auc": 0.82604, "rw": 0.82604, "b0_24": 0.82604, "b24_48": NaN, "b48p": NaN, "d8_auc": 0.00415, "d8_rw": 0.00415, "d10_auc": 0.00298, "d10_rw": 0.00298, "d10_b0_24": 0.00298, "d10_b24_48": NaN, "d10_b48p": NaN} +{"name": "v11_seed1", "spec": "sCbig:1:1+sC_s:1:1+2*mlp2:3:1", "cut": 25, "auc": 0.81523, "rw": 0.81523, "b0_24": 0.81867, "b24_48": 0.80089, "b48p": NaN, "d8_auc": 0.00202, "d8_rw": 0.00202, "d10_auc": 0.00135, "d10_rw": 0.00135, "d10_b0_24": 0.00128, "d10_b24_48": 0.00166, "d10_b48p": NaN} +{"name": "v11_seed1", "spec": "sCbig:1:1+sC_s:1:1+2*mlp2:3:1", "cut": 23, "auc": 0.79424, "rw": 0.79308, "b0_24": 0.80343, "b24_48": 0.77497, "b48p": 0.76729, "d8_auc": 0.00368, "d8_rw": 0.00368, "d10_auc": 0.00322, "d10_rw": 0.00324, "d10_b0_24": 0.00305, "d10_b24_48": 0.00355, "d10_b48p": 0.00373} +{"name": "b_seed1", "spec": "dwbig:1:1+dw_s:1:1+2*dwmlp:3:1", "cut": 25, "auc": 0.81721, "rw": 0.8172, "b0_24": 0.82064, "b24_48": 0.80288, "b48p": NaN, "d8_auc": 0.004, "d8_rw": 0.004, "d10_auc": 0.00333, "d10_rw": 0.00333, "d10_b0_24": 0.00324, "d10_b24_48": 0.00366, "d10_b48p": NaN} +{"name": "b_seed1", "spec": "dwbig:1:1+dw_s:1:1+2*dwmlp:3:1", "cut": 23, "auc": 0.79557, "rw": 0.7944, "b0_24": 0.8047, "b24_48": 0.77696, "b48p": 0.7681, "d8_auc": 0.00502, "d8_rw": 0.005, "d10_auc": 0.00456, "d10_rw": 0.00456, "d10_b0_24": 0.00432, "d10_b24_48": 0.00553, "d10_b48p": 0.00454} +{"name": "v11_lgbs1_mlp012", "spec": "sCbig:1:1+sC_s:1:1+2*mlp2:3:0,1,2", "cut": 25, "auc": 0.81529, "rw": 0.81529, "b0_24": 0.81873, "b24_48": 0.80094, "b48p": NaN, "d8_auc": 0.00208, "d8_rw": 0.00208, "d10_auc": 0.00141, "d10_rw": 0.00141, "d10_b0_24": 0.00134, "d10_b24_48": 0.00172, "d10_b48p": NaN} +{"name": "v11_lgbs1_mlp012", "spec": "sCbig:1:1+sC_s:1:1+2*mlp2:3:0,1,2", "cut": 23, "auc": 0.79423, "rw": 0.79308, "b0_24": 0.80342, "b24_48": 0.77504, "b48p": 0.7672, "d8_auc": 0.00368, "d8_rw": 0.00367, "d10_auc": 0.00322, "d10_rw": 0.00324, "d10_b0_24": 0.00304, "d10_b24_48": 0.00362, "d10_b48p": 0.00364} +{"name": "b_lgbs1_mlp012", "spec": "dwbig:1:1+dw_s:1:1+2*dwmlp:3:0,1,2", "cut": 25, "auc": 0.81727, "rw": 0.81727, "b0_24": 0.82069, "b24_48": 0.803, "b48p": NaN, "d8_auc": 0.00406, "d8_rw": 0.00406, "d10_auc": 0.00339, "d10_rw": 0.00339, "d10_b0_24": 0.00329, "d10_b24_48": 0.00378, "d10_b48p": NaN} +{"name": "b_lgbs1_mlp012", "spec": "dwbig:1:1+dw_s:1:1+2*dwmlp:3:0,1,2", "cut": 23, "auc": 0.79562, "rw": 0.79444, "b0_24": 0.80474, "b24_48": 0.77695, "b48p": 0.76822, "d8_auc": 0.00506, "d8_rw": 0.00504, "d10_auc": 0.0046, "d10_rw": 0.0046, "d10_b0_24": 0.00436, "d10_b24_48": 0.00552, "d10_b48p": 0.00466} +{"name": "v11_lgb01", "spec": "sCbig:1:0,1+sC_s:1:0,1+2*mlp2:3:0,1,2", "cut": 25, "auc": 0.81529, "rw": 0.81529, "b0_24": 0.81873, "b24_48": 0.80094, "b48p": NaN, "d8_auc": 0.00208, "d8_rw": 0.00208, "d10_auc": 0.00141, "d10_rw": 0.00141, "d10_b0_24": 0.00134, "d10_b24_48": 0.00171, "d10_b48p": NaN} +{"name": "v11_lgb01", "spec": "sCbig:1:0,1+sC_s:1:0,1+2*mlp2:3:0,1,2", "cut": 23, "auc": 0.79429, "rw": 0.79313, "b0_24": 0.80347, "b24_48": 0.77516, "b48p": 0.76717, "d8_auc": 0.00373, "d8_rw": 0.00372, "d10_auc": 0.00327, "d10_rw": 0.00329, "d10_b0_24": 0.00309, "d10_b24_48": 0.00374, "d10_b48p": 0.00361} +{"name": "b_lgb01", "spec": "dwbig:1:0,1+dw_s:1:0,1+2*dwmlp:3:0,1,2", "cut": 25, "auc": 0.81728, "rw": 0.81728, "b0_24": 0.82071, "b24_48": 0.80296, "b48p": NaN, "d8_auc": 0.00407, "d8_rw": 0.00407, "d10_auc": 0.0034, "d10_rw": 0.0034, "d10_b0_24": 0.00332, "d10_b24_48": 0.00373, "d10_b48p": NaN} +{"name": "b_lgb01", "spec": "dwbig:1:0,1+dw_s:1:0,1+2*dwmlp:3:0,1,2", "cut": 23, "auc": 0.79568, "rw": 0.79451, "b0_24": 0.80479, "b24_48": 0.77704, "b48p": 0.76836, "d8_auc": 0.00513, "d8_rw": 0.00511, "d10_auc": 0.00467, "d10_rw": 0.00467, "d10_b0_24": 0.00441, "d10_b24_48": 0.00562, "d10_b48p": 0.0048} diff --git a/v13/e.sh b/v13/e.sh new file mode 100644 index 0000000000000000000000000000000000000000..14229357bb4240610e87ff0bf208632744574853 --- /dev/null +++ b/v13/e.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# usage: e.sh TAG "CUTS" ROUNDS SAVE NEGFRAC PARAMSJSON [LRDIRS] [extra ens args] (8 threads, v10B drop) +cd /workspace/huawei/v13; P=/workspace/venv/bin/python; TAG=$1; CUTS=$2; R=$3; SV=$4; NF=$5; PJ=$6; LR=${7:-feat_lr2,feat_tt,feat_pp,feat_sq10}; shift $(( $# < 7 ? $# : 7 )) +export POLARS_MAX_THREADS=8 OMP_NUM_THREADS=8 PYTHONUNBUFFERED=1 +for c in $CUTS; do + t0=$(date +%s) + $P ens.py $c $TAG --seeds ${SEEDS:-0} --threads 8 --lr $LR --negfrac $NF --nfolds ${NFOLDS:-4} --rounds $R --save $SV --params "$PJ" --drop '^sq_(pdot|pcos|mdot|mcos)$' "$@" 2>&1 | grep -v Warn | tail -3 + echo "{\"tag\":\"$TAG\",\"cut\":$c,\"wall_s\":$(( $(date +%s)-t0 ))}" >> timings.jsonl +done diff --git a/v13/ens.py b/v13/ens.py new file mode 100644 index 0000000000000000000000000000000000000000..a9161a4343f92cba8daa4a4a244b84fd69e07522 --- /dev/null +++ b/v13/ens.py @@ -0,0 +1,41 @@ +"""Phase 1 ensemble components. python ens.py CUT TAG [--seeds 0,1] [--folds f|fe] [--stale] [--rc] [--drop re] [--params json] [--rounds 400] +CUT=25/26: train f1..f(CUT-21) (+e/d folds) full data, predict val{CUT}; CUT=27: train f1..f6 (+E1..E6), predict eval. +Saves ens/{tag}_c{CUT}_s{seed}.npy with raw scores at rounds 300,400(,500) (columns).""" +import sys, re, json, gc, time, argparse, numpy as np, lightgbm as lgb +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--seeds',default='0') +ap.add_argument('--folds',default='f'); ap.add_argument('--stale',action='store_true'); ap.add_argument('--rc',action='store_true') +ap.add_argument('--drop',default=None); ap.add_argument('--params',default='{}'); ap.add_argument('--rounds',type=int,default=400); ap.add_argument('--threads',type=int,default=5); ap.add_argument('--lr',default=''); ap.add_argument('--nfolds',type=int,default=99); ap.add_argument('--negfrac',type=float,default=1.0); ap.add_argument('--keep',default=''); ap.add_argument('--wtag',default=''); ap.add_argument('--wpow',type=float,default=1.0); ap.add_argument('--agew',type=float,default=0); ap.add_argument('--save',default='300,400,500') +a=ap.parse_args(); t0=time.time() +extra=(('feat_rc',) if a.rc else ())+tuple(x for x in a.lr.split(',') if x); var='stale' if a.stale else 'notime' +efold={23:[],25:[f'e{k}' for k in range(1,5)],26:[f'd{k}' for k in range(1,6)],27:[f'E{k}' for k in range(1,7)]}[a.cut] +trn=([f'f{k}' for k in range(max(1,a.cut-20-a.nfolds),a.cut-20)] if 'f' in a.folds else [])+(efold if 'e' in a.folds else []) +import polars as pl +lens=[pl.read_parquet_schema and len(pl.read_parquet(f'feat_v3/{n}.parquet',columns=['rowid'])) for n in trn] +rs=np.random.RandomState(123); R=rs.rand(sum(lens)) if a.negfrac<1 else None +Xs=[];ys=[];ws=[];feats=None;off=0 +for n,L in zip(trn,lens): + X,m,f=matrix(n,1.0,variant=var,extra=extra); feats=f; yy=m['clicked'].to_numpy(); assert len(yy)==L + sel=(yy==1)|(R[off:off+L]2 else 0; t0=time.time() +X,m,fl=matrix(f'val{cut}',1.0,extra=BASE+('feat_sqC','feat_dw')); y=m['clicked'].to_numpy(); z=z12(cut) +user=pl.read_parquet(f'{R.F3}/val{cut}.parquet',columns=['u'])['u'].to_numpy() +n=len(y)//10; X3=X.reshape(n,10,-1) +const=(np.nanmax(X3,1)-np.nanmin(X3,1)==0).all(0)|np.isnan(X3[:,0,:]).all(0) +V=X3[:,:,~const]; vn=[c for c,k in zip(fl,~const) if k] +hl=R.hist_len(m['rowid'].to_numpy()); zz,dm,rk=R.wev(z); Z=z.reshape(-1,10) +Xb=np.column_stack([z,zz,dm,rk,np.repeat(np.log(np.exp(Z).sum(1)),10),np.repeat(Z.max(1),10),np.repeat(Z.mean(1),10),hl,(hl==0)]).astype(np.float32) +yc=(y.reshape(n,10)-y.reshape(n,10).mean(1,keepdims=True)).ravel() +Vd=np.nan_to_num(V-np.nanmean(V,1,keepdims=True)).reshape(n*10,-1).astype(np.float32) +sc=np.abs((Vd*yc[:,None]).sum(0))/(np.sqrt((Vd**2).sum(0))+1e-9); top=np.argsort(-sc)[:80]; print('top',[vn[i] for i in top[:15]],flush=True) +V=V[:,:,top]; RD=Vd[:,top]; del Vd +C=np.repeat(X3[:,0,const],10,0) +agg=lambda f:np.repeat(f(V,1),10,0) +EA=np.hstack([C,agg(np.nanmean),agg(np.nanmax),agg(np.nanmin),agg(np.nanstd)]).astype(np.float32) +sets={'base':Xb,'+evconst+agg':np.hstack([Xb,EA]),'+rowdev':np.hstack([Xb,RD]),'+all':np.hstack([Xb,EA,RD])} +print('built',{k:v.shape for k,v in sets.items()},time.time()-t0,flush=True) +res={} +for name,F in sets.items(): + p=np.zeros(len(y)) + for tr,te in GroupKFold(5).split(F,y,user): p[te]=R.fit_pred(F[tr],y[tr],z[tr],F[te],z[te],seed) + res[name]=A(y,p); r=dict(cut=cut,seed=seed,set=name,v12=A(y,z),auc=res[name],d_vs_v12=res[name]-A(y,z),d_vs_base=res[name]-res.get('base',res[name])) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open('exp_results.jsonl','a').write(json.dumps(r)+'\n') +print('t',time.time()-t0) diff --git a/v13/exp_results.jsonl b/v13/exp_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4c73a00c2e3520739f5ce2bd64ccae06c188e7a9 --- /dev/null +++ b/v13/exp_results.jsonl @@ -0,0 +1,4 @@ +{"cut": 25, "seed": 0, "set": "base", "v12": 0.8172259475835104, "auc": 0.817259094679424, "d_vs_v12": 3.314709591362153e-05, "d_vs_base": 0.0} +{"cut": 25, "seed": 0, "set": "+evconst+agg", "v12": 0.8172259475835104, "auc": 0.8175111305947576, "d_vs_v12": 0.0002851830112471365, "d_vs_base": 0.000252035915333515} +{"cut": 25, "seed": 0, "set": "+rowdev", "v12": 0.8172259475835104, "auc": 0.8175529011949488, "d_vs_v12": 0.0003269536114384186, "d_vs_base": 0.0002938065155247971} +{"cut": 25, "seed": 0, "set": "+all", "v12": 0.8172259475835104, "auc": 0.8176341641907765, "d_vs_v12": 0.00040821660726608755, "d_vs_base": 0.000375069511352466} diff --git a/v13/logs/b1.log b/v13/logs/b1.log new file mode 100644 index 0000000000000000000000000000000000000000..7422cd705a2990ad03ebf1c931b23b525f3be21a --- /dev/null +++ b/v13/logs/b1.log @@ -0,0 +1,3 @@ +{"cut": 25, "tag": "ev_s", "seed": 0, "n": 745456, "nf": 835, "min": 2.8, "load_s": 14, "ds_s": 10, "fit_s": 137, "pred_s": 4, "auc": {"400": 0.81594, "500": 0.81593}} +{"cut": 23, "tag": "ev_s", "seed": 0, "n": 373643, "nf": 835, "min": 0.9, "load_s": 6, "ds_s": 6, "fit_s": 35, "pred_s": 5, "auc": {"400": 0.79431, "500": 0.79418}} +B1_DONE diff --git a/v13/logs/b3.log b/v13/logs/b3.log new file mode 100644 index 0000000000000000000000000000000000000000..d257071ecc8c00936eaac18eb040b76789558193 --- /dev/null +++ b/v13/logs/b3.log @@ -0,0 +1,5 @@ +weights: mean 1.0000001 p01/p50/p99 [0.5420776 0.77439666 1.5804013 ] ESS frac 0.8679212 +{"cut": 25, "tag": "aw_s", "seed": 0, "n": 745456, "nf": 435, "min": 0.9, "load_s": 6, "ds_s": 4, "fit_s": 36, "pred_s": 4, "auc": {"400": 0.81601, "500": 0.81589}} +weights: mean 1.0000001 p01/p50/p99 [0.82354546 0.82354546 1.17649353] ESS frac 0.9697976 +{"cut": 23, "tag": "aw_s", "seed": 0, "n": 373643, "nf": 435, "min": 0.5, "load_s": 3, "ds_s": 3, "fit_s": 20, "pred_s": 5, "auc": {"400": 0.79413, "500": 0.7938}} +B3_DONE diff --git a/v13/logs/exp1_25.log b/v13/logs/exp1_25.log new file mode 100644 index 0000000000000000000000000000000000000000..fb93456a6b133e7e34f5131819ada2be382ad1ca --- /dev/null +++ b/v13/logs/exp1_25.log @@ -0,0 +1,7 @@ +v12 0.8172259475835104 +event-const cols 60 of 439 +event AUC of e0 0.672879467007552 lse 0.6680507246702352 +{"cut": 25, "set": "zstats", "ev_auc_e0": 0.67288, "ev_auc_new": 0.67131, "glob_a0.5": -0.00136, "glob_a1.0": -0.00488} +{"cut": 25, "set": "zstats+const", "ev_auc_e0": 0.67288, "ev_auc_new": 0.67419, "glob_a0.5": -0.00125, "glob_a1.0": -0.00531} +{"cut": 25, "set": "zstats+const+agg", "ev_auc_e0": 0.67288, "ev_auc_new": 0.67466, "glob_a0.5": -0.00132, "glob_a1.0": -0.00562} +t 36.13779139518738 diff --git a/v13/logs/exp2_25.log b/v13/logs/exp2_25.log new file mode 100644 index 0000000000000000000000000000000000000000..06c304a39eba94200750ebe3c4583a5b73492080 --- /dev/null +++ b/v13/logs/exp2_25.log @@ -0,0 +1,7 @@ +top ['tB_dot_z', 'tB_dmean', 'tB_dot', 'tA_cos', 'tA_dmax', 'tA_z', 'lr10_dmax', 'lr10_cos', 'lr12_con05', 'lr12_con05_dmax', 'lr12_dmax', 'lr12_cos', 'sq_gap2', 'sq_gap', 'sq_logit'] +built {'base': (439640, 9), '+evconst+agg': (439640, 389), '+rowdev': (439640, 89), '+all': (439640, 469)} 11.77814793586731 +{"cut": 25, "seed": 0, "set": "base", "v12": 0.81723, "auc": 0.81726, "d_vs_v12": 3e-05, "d_vs_base": 0.0} +{"cut": 25, "seed": 0, "set": "+evconst+agg", "v12": 0.81723, "auc": 0.81751, "d_vs_v12": 0.00029, "d_vs_base": 0.00025} +{"cut": 25, "seed": 0, "set": "+rowdev", "v12": 0.81723, "auc": 0.81755, "d_vs_v12": 0.00033, "d_vs_base": 0.00029} +{"cut": 25, "seed": 0, "set": "+all", "v12": 0.81723, "auc": 0.81763, "d_vs_v12": 0.00041, "d_vs_base": 0.00038} +t 145.99419975280762 diff --git a/v13/logs/mkev.log b/v13/logs/mkev.log new file mode 100644 index 0000000000000000000000000000000000000000..0591838115c4697b37e99596ae7131cdedebb5e8 --- /dev/null +++ b/v13/logs/mkev.log @@ -0,0 +1,11 @@ +['tB_dmean', 'tB_dot', 'tB_dot_z', 'tA_cos', 'tA_dmax', 'lr12_dmax', 'lr12_cos', 'tA_z', 'lr16_dmax', 'lr16_cos', 'lr12_con05', 'lr12_con05_dmax', 'lr12_z', 'lr16_con05_dmax', 'lr16_con05', 'lr12_con05_z', 'lr10_dmax', 'lr10_cos', 'c_hist', 'lr16_z'] +val25 439640 8.138801574707031 +val23 501550 8.708249568939209 +val26 351380 5.456751823425293 +eval 529390 7.82628607749939 +f1 351560 4.5169031620025635 +f2 351120 6.267075777053833 +f3 349120 5.32174277305603 +f4 350400 5.697640657424927 +f5 352300 5.685654401779175 +f6 351380 5.352286338806152 diff --git a/v13/logs/mkev1.log b/v13/logs/mkev1.log new file mode 100644 index 0000000000000000000000000000000000000000..eacb22317fa684bc536643ed0a94817fc0000fa6 --- /dev/null +++ b/v13/logs/mkev1.log @@ -0,0 +1 @@ +['tB_dmean', 'tB_dot', 'tB_dot_z', 'tA_cos', 'tA_dmax', 'lr12_dmax', 'lr12_cos', 'tA_z', 'lr16_dmax', 'lr16_cos', 'lr12_con05', 'lr12_con05_dmax', 'lr12_z', 'lr16_con05_dmax', 'lr16_con05', 'lr12_con05_z', 'lr10_dmax', 'lr10_cos', 'c_hist', 'lr16_z'] diff --git a/v13/mkev.py b/v13/mkev.py new file mode 100644 index 0000000000000000000000000000000000000000..0243c327f8cd8e634b6e03e89269ac8f13f18f4a --- /dev/null +++ b/v13/mkev.py @@ -0,0 +1,26 @@ +"""feat_ev: event-slate aggregates (mean/max/min/std over the 10 own-slate rows) + row deviation for top-K features. +K selected by within-event label corr on f1 (train fold, earlier than every cut's eval). Uses only each table's own slates.""" +import sys, json, os, time, numpy as np, polars as pl, gc +from cl import matrix, BASE +K=int(os.environ.get('K',80)); os.makedirs('feat_ev',exist_ok=True) +EX=BASE+('feat_sqC','feat_dw') +def sel(): + if os.path.exists('feat_ev/top.json'): return json.load(open('feat_ev/top.json')) + X,m,fl=matrix('f1',1.0,extra=EX); y=m['clicked'].to_numpy().astype(np.float32); n=len(y)//10 + X3=X.reshape(n,10,-1); const=(np.nanmax(X3,1)-np.nanmin(X3,1)==0).all(0)|np.isnan(X3[:,0,:]).all(0) + yc=(y.reshape(n,10)-y.reshape(n,10).mean(1,keepdims=True)).ravel(); sc=np.zeros(X.shape[1]) + for j in range(X.shape[1]): + if const[j]: continue + v=X3[:,:,j]; d=np.nan_to_num(v-np.nanmean(v,1,keepdims=True)).ravel(); sc[j]=abs((d*yc).sum())/(np.sqrt((d**2).sum())+1e-9) + top=[fl[i] for i in np.argsort(-sc)[:K]]; json.dump(top,open('feat_ev/top.json','w')); return top +top=sel(); print(top[:20],flush=True) +for name in sys.argv[1:]: + t0=time.time(); X,m,fl=matrix(name,1.0,extra=EX); idx=[fl.index(c) for c in top]; V=X[:,idx].reshape(-1,10,len(idx)); del X; gc.collect() + mu=np.nanmean(V,1,keepdims=True); mx=np.nanmax(V,1,keepdims=True); mn=np.nanmin(V,1,keepdims=True); sd=np.nanstd(V,1,keepdims=True) + rep=lambda a:np.repeat(a[:,0,:],10,0) + mu,mx,mn,sd=rep(mu),rep(mx),rep(mn),rep(sd); mu3=mu + cols={'rowid':m['rowid'].to_numpy()} + for k,c in enumerate(top): + cols[f'ev_mean_{c}']=mu[:,k].copy(); cols[f'ev_max_{c}']=mx[:,k].copy(); cols[f'ev_min_{c}']=mn[:,k].copy(); cols[f'ev_std_{c}']=sd[:,k].copy() + cols[f'ev_dm_{c}']=(V[:,:,k].ravel()-mu[:,k]) + pl.DataFrame({k:(v.astype(np.float32) if k!='rowid' else v) for k,v in cols.items()}).write_parquet(f'feat_ev/{name}.parquet'); print(name,len(m),time.time()-t0,flush=True) diff --git a/v13/mksub.py b/v13/mksub.py new file mode 100644 index 0000000000000000000000000000000000000000..aebf558c64a8e6d134091dd1792c500b2a035a91 --- /dev/null +++ b/v13/mksub.py @@ -0,0 +1,20 @@ +"""mksub.py OUT SPEC : spec like cmp11 (tag:col items joined by '+', optional w*), cut 27 -> sigmoid(mean score) -> OUT csv (rowid,p) ordered as v8 reference.""" +import sys, numpy as np, polars as pl, pandas as pd +from cl import matrix +out, spec = sys.argv[1], sys.argv[2] +from cl import BASE +_, m, _ = matrix('eval', 1.0, extra=BASE) +def load(item): + w=1.0 + if '*' in item: w,item=item.split('*'); w=float(w) + p=item.split(':'); sd=p[2].split(',') if len(p)>2 else ['0']; return w*np.mean([np.load(f'ens/{p[0]}_c27_s{x}.npy')[:,int(p[1])] for x in sd],0) +it=spec.split('+'); W=sum(float(i.split('*')[0]) if '*' in i else 1.0 for i in it); z=np.sum([load(i) for i in it],0)/W +assert len(z)==m.height; p=1/(1+np.exp(-z)); assert np.isfinite(p).all() and (p>=0).all() and (p<=1).all() +df=pd.DataFrame({'rowid':m['rowid'].to_numpy(),'p':p}) +ref=pd.read_csv('/workspace/huawei/v2work/out/submission_v8.csv'); print(ref.columns.tolist(), len(ref)) +assert df.rowid.is_unique and len(df)==529390 and set(df.rowid)==set(ref.iloc[:,0]), 'rowid set mismatch' +df=df.set_index('rowid').loc[ref.iloc[:,0]].reset_index(); df.columns=list(ref.columns) +df.to_csv(out,index=False); print('wrote',out,len(df),'p range',p.min(),p.max(),'mean',p.mean()) +from scipy.stats import spearmanr +v10=pd.read_csv('/workspace/huawei/v10/out/submission_v10.csv'); a=df.set_index(df.columns[0]).iloc[:,0]; b=v10.set_index(v10.columns[0]).iloc[:,0].loc[a.index] +print('spearman vs v10', spearmanr(a,b)[0]); v8=ref.set_index(ref.columns[0]).iloc[:,0].loc[a.index]; print('spearman vs v8', spearmanr(a,v8)[0]) diff --git a/v13/mlp11.py b/v13/mlp11.py new file mode 100644 index 0000000000000000000000000000000000000000..0eb9bb3f63daebcb1e510f7e2e5c504555842ec3 --- /dev/null +++ b/v13/mlp11.py @@ -0,0 +1,42 @@ +"""mlp11.py CUT TAG [--ep 4] [--seed 0]: MLP on the same LightGBM matrix (train f(C-24)..f(C-21), negfrac .5, v10B drop), rank-gauss inputs +from TRAIN quantiles only + missing flags. Saves ens/{TAG}_c{CUT}_s{seed}.npy (n,EP) logits after each epoch.""" +import sys, re, time, argparse, json, gc, numpy as np, torch, torch.nn as nn +from cl import matrix +from sklearn.metrics import roc_auc_score +ap=argparse.ArgumentParser(); ap.add_argument('cut',type=int); ap.add_argument('tag'); ap.add_argument('--ep',type=int,default=4); ap.add_argument('--seed',type=int,default=0) +ap.add_argument('--lr',default='feat_lr2,feat_tt,feat_pp,feat_sqA'); ap.add_argument('--hid',type=int,default=512); ap.add_argument('--drop',type=float,default=0.2) +a=ap.parse_args(); t0=time.time(); torch.manual_seed(a.seed); np.random.seed(a.seed); torch.set_num_threads(8) +DR='^sq_(pdot|pcos|mdot|mcos)$'; extra=tuple(a.lr.split(',')) +trn=[f'f{k}' for k in range(max(1,a.cut-24),a.cut-20)] +rs=np.random.RandomState(123); Xs=[];ys=[] +for n in trn: + X,m,f=matrix(n,1.0,extra=extra); y=m['clicked'].to_numpy(); sel=(y==1)|(rs.rand(len(y))<0.5); Xs.append(X[sel]); ys.append(y[sel]); feats=f; del X; gc.collect() +keep=[i for i,c in enumerate(feats) if not re.search(DR,c)]; fl=[feats[i] for i in keep] +X=np.concatenate([x[:,keep] for x in Xs]); del Xs; y=np.concatenate(ys).astype(np.float32); gc.collect() +vn='eval' if a.cut==27 else f'val{a.cut}'; Xv,mv,fv=matrix(vn,1.0,extra=extra); Xv=Xv[:,[fv.index(c) for c in fl]] +# rank-gauss via train quantiles (101 knots); NaN -> 0 + flag +from scipy.special import ndtri +qs=np.linspace(0.001,0.999,101); Q=np.nanquantile(X[::7],qs,axis=0).astype(np.float32) # (101,F) +nanc=np.where(np.isnan(X[::7]).any(0)|np.isnan(Xv[::7]).any(0))[0]; G=ndtri(qs).astype(np.float32) +def tr(Z): + out=np.empty((len(Z),Z.shape[1]+len(nanc)),np.float32) + for j in range(Z.shape[1]): + q=np.nan_to_num(Q[:,j],nan=0.0); v=Z[:,j]; u,ix=np.unique(q,return_index=True) + o=np.interp(v,u,G[ix]) if len(u)>1 else np.zeros_like(v); o[np.isnan(v)]=0; out[:,j]=o + out[:,Z.shape[1]:]=np.isnan(Z[:,nanc]); return out +X=np.nan_to_num(tr(X)); Xv=np.nan_to_num(tr(Xv)); gc.collect(); print('prep',X.shape,round(time.time()-t0),'s',flush=True) +D=X.shape[1] +net=nn.Sequential(nn.Linear(D,a.hid),nn.BatchNorm1d(a.hid),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid,a.hid//2),nn.BatchNorm1d(a.hid//2),nn.SiLU(),nn.Dropout(a.drop),nn.Linear(a.hid//2,1)) +opt=torch.optim.AdamW(net.parameters(),1e-3,weight_decay=1e-4); lf=nn.BCEWithLogitsLoss(); Xt=torch.from_numpy(X); Yt=torch.from_numpy(y); Xvt=torch.from_numpy(Xv) +bs=2048; steps=a.ep*((len(y)+bs-1)//bs); sch=torch.optim.lr_scheduler.OneCycleLR(opt,2e-3,total_steps=steps,pct_start=0.1); S=[] +for ep in range(a.ep): + net.train(); perm=torch.randperm(len(y)) + for b in range(0,len(y),bs): + j=perm[b:b+bs]; opt.zero_grad(); l=lf(net(Xt[j]).squeeze(-1),Yt[j]); l.backward(); opt.step(); sch.step() + net.eval() + with torch.no_grad(): p=torch.cat([net(Xvt[b:b+65536]).squeeze(-1) for b in range(0,len(Xv),65536)]).numpy() + S.append(p); msg=f'ep{ep} {time.time()-t0:.0f}s' + if a.cut!=27: msg+=f' auc {roc_auc_score(mv["clicked"].to_numpy(),p):.5f}' + print(msg,flush=True) +np.save(f'ens/{a.tag}_c{a.cut}_s{a.seed}.npy',np.stack(S,1).astype(np.float32)) +open('ens/ens_log.jsonl','a').write(json.dumps(dict(cut=a.cut,tag=a.tag,seed=a.seed,min=round((time.time()-t0)/60,1)))+'\n') diff --git a/v13/resid.py b/v13/resid.py new file mode 100644 index 0000000000000000000000000000000000000000..362f62697dddc3f2679ffbdf2cc6d9fb01553efc --- /dev/null +++ b/v13/resid.py @@ -0,0 +1,62 @@ +"""resid.py: method (a) = DW residual boosting on top of v11 logits (init_score), features = DW reusable 42 cols + v11-derived context. +modes: oof (DW protocol: per cut 5-fold GroupKFold by user, seeds) ; xcut (train on the other val cuts' rows, users of the test cut removed -> predict test cut; mimics eval application) + final (train on pooled val23/25/26 rows, predict eval; saves ens/resid_c27_s{seed}.npy as logit)""" +import sys, json, numpy as np, polars as pl, lightgbm as lgb, warnings; warnings.filterwarnings('ignore') +from sklearn.metrics import roc_auc_score as A +from sklearn.model_selection import GroupKFold +W='/workspace/huawei/v13'; V11='/workspace/huawei/v11'; F3='/workspace/huawei/v10/feat_v3' +PI=dict(objective='binary',learning_rate=0.03,num_leaves=15,min_data_in_leaf=500,feature_fraction=0.8,bagging_fraction=0.8,bagging_freq=1,lambda_l2=30,max_bin=127,verbose=-1,num_threads=8) +def v11z(c): + a=np.load(f'{V11}/ens/sCbig_c{c}_s0.npy')[:,1]; b=np.load(f'{V11}/ens/sC_s_c{c}_s0.npy')[:,1] + m=np.mean([np.load(f'{V11}/ens/mlp2_c{c}_s{k}.npy')[:,3] for k in (range(3) if c!=26 else [0])],0); return ((a+b+2*m)/4).astype(np.float64) +def wev(x): + m=x.reshape(-1,10); mu=m.mean(1,keepdims=True); sd=m.std(1,keepdims=True)+1e-6; return ((m-mu)/sd).ravel(),(m-mu).ravel(),m.argsort(1).argsort(1).astype(np.float32).ravel() +_HL=None +def hist_len(rowid): + global _HL + if _HL is None: + L='/workspace/huawei/lite' + te=pl.concat([pl.read_parquet(f'{L}/train_events.parquet',columns=['eid','h_items']),pl.read_parquet(f'{L}/evaluation_events.parquet',columns=['eid','h_items'])]) + rw=pl.concat([pl.read_parquet(f'{L}/train_rows.parquet',columns=['rowid','eid']),pl.read_parquet(f'{L}/evaluation_rows.parquet',columns=['rowid','eid'])]) + _HL=rw.join(te.select('eid',pl.col('h_items').list.len().alias('hl')),on='eid',how='left') + return pl.DataFrame({'rowid':rowid}).join(_HL.select('rowid','hl'),on='rowid',how='left',maintain_order='left')['hl'].to_numpy().astype(np.float32) +def frame(c,use_dw=True): + n='eval' if c==27 else f'val{c}'; m=pl.read_parquet(f'{F3}/{n}.parquet',columns=['rowid']+([] if c==27 else ['clicked'])) + z=v11z(c); assert len(z)==m.height; zz,dm,rk=wev(z); Z=z.reshape(-1,10); hl=hist_len(m['rowid'].to_numpy()) + B={'v11':z,'v11_z':zz,'v11_dm':dm,'v11_rk':rk,'ev_lse':np.repeat(np.log(np.exp(Z).sum(1)),10),'ev_max':np.repeat(Z.max(1),10),'ev_mean':np.repeat(Z.mean(1),10),'hist_len':hl,'nohist':(hl==0).astype(np.float32)} + d=pl.read_parquet(f'{W}/feat_dw/{n}.parquet'); assert (d['rowid']==m['rowid']).all() + Xb=np.column_stack(list(B.values())).astype(np.float32); Xd=d.drop('rowid').to_numpy().astype(np.float32) + y=m['clicked'].to_numpy() if c!=27 else None + user=pl.read_parquet(f'{F3}/{n}.parquet',columns=['u'])['u'].to_numpy() if 'u' in pl.read_parquet_schema(f'{F3}/{n}.parquet') else None + return dict(Xb=Xb,Xd=Xd,y=y,z=z,user=user,cut=c) +def fit_pred(Xtr,ytr,itr,Xte,ite,seed,rounds=200): + m=lgb.train({**PI,'seed':seed},lgb.Dataset(Xtr,ytr,init_score=itr),rounds); return m.predict(Xte,raw_score=True)+ite +if __name__=='__main__': + mode=sys.argv[1]; seeds=[int(s) for s in sys.argv[2].split(',')]; cuts=[int(c) for c in sys.argv[3].split(',')] if len(sys.argv)>3 else [25,23,26] + FR={c:frame(c) for c in set(cuts+[25,23,26])} + for c in FR: + if FR[c]['user'] is None: raise SystemExit('no user col') + if mode=='oof': + for c in cuts: + f=FR[c]; y=f['y']; res={} + for seed in seeds: + p0=np.zeros(len(y)); p1=np.zeros(len(y)) + for tr,te in GroupKFold(5).split(f['Xb'],y,f['user']): + p0[te]=fit_pred(f['Xb'][tr],y[tr],f['z'][tr],f['Xb'][te],f['z'][te],seed) + X=np.hstack([f['Xb'],f['Xd']]); p1[te]=fit_pred(X[tr],y[tr],f['z'][tr],X[te],f['z'][te],seed) + r=dict(mode='oof',cut=c,seed=seed,v11=A(y,f['z']),base=A(y,p0),new=A(y,p1)); r['d_vs_v11']=r['new']-r['v11']; r['d_vs_base']=r['new']-r['base'] + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open(f'{W}/resid_results.jsonl','a').write(json.dumps(r)+'\n') + elif mode=='xcut': + for c in cuts: + oth=[o for o in (25,23,26) if o!=c]; te=FR[c]; ute=set(te['user'].tolist()) + tr=[FR[o] for o in oth]; keep=[~np.isin(t['user'],list(ute)) for t in tr] + Xtr=np.vstack([np.hstack([t['Xb'],t['Xd']])[k] for t,k in zip(tr,keep)]); ytr=np.concatenate([t['y'][k] for t,k in zip(tr,keep)]); itr=np.concatenate([t['z'][k] for t,k in zip(tr,keep)]) + Xtr_b=Xtr[:,:te['Xb'].shape[1]]; Xte=np.hstack([te['Xb'],te['Xd']]) + for seed in seeds: + p0=fit_pred(Xtr_b,ytr,itr,te['Xb'],te['z'],seed); p1=fit_pred(Xtr,ytr,itr,Xte,te['z'],seed) + r=dict(mode='xcut',cut=c,seed=seed,ntr=int(len(ytr)),v11=A(te['y'],te['z']),base=A(te['y'],p0),new=A(te['y'],p1)); r['d_vs_v11']=r['new']-r['v11']; r['d_vs_base']=r['new']-r['base'] + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open(f'{W}/resid_results.jsonl','a').write(json.dumps(r)+'\n') + elif mode=='final': + te=frame(27); tr=[FR[o] for o in (25,23,26)] + Xtr=np.vstack([np.hstack([t['Xb'],t['Xd']]) for t in tr]); ytr=np.concatenate([t['y'] for t in tr]); itr=np.concatenate([t['z'] for t in tr]); Xte=np.hstack([te['Xb'],te['Xd']]) + P=np.mean([fit_pred(Xtr,ytr,itr,Xte,te['z'],s) for s in seeds],0); np.save(f'{W}/ens/resid_c27.npy',P.astype(np.float32)); print('saved',P.shape) diff --git a/v13/ship.sh b/v13/ship.sh new file mode 100644 index 0000000000000000000000000000000000000000..befafccc3d0c87c5c8838e72dee0ab7deffeb29c --- /dev/null +++ b/v13/ship.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# ship.sh AUC25 RW : gate + candidate zip + sha +cd /workspace/huawei/v13 +python3 /workspace/huawei/tools/package_final.py --csv /workspace/huawei/v13/out/submission_v13.csv --auc25 $1 --rw25 $2 --name v13; echo gate_exit=$? +cd out; rm -f submission_v13_candidate.csv.zip; /workspace/venv/bin/python -c " +import zipfile +z=zipfile.ZipFile('submission_v13_candidate.csv.zip','w',zipfile.ZIP_DEFLATED); z.write('submission_v13.csv',arcname='submission_v13.csv')" +sha256sum submission_v13_candidate.csv.zip submission_v13.csv | tee submission_v13_candidate.csv.zip.sha256 diff --git a/v13/timings.jsonl b/v13/timings.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ca0832d4a5809390fb0c173120a718b3e642376f --- /dev/null +++ b/v13/timings.jsonl @@ -0,0 +1,4 @@ +{"tag":"ev_s","cut":25,"wall_s":168} +{"tag":"ev_s","cut":23,"wall_s":55} +{"tag":"aw_s","cut":25,"wall_s":53} +{"tag":"aw_s","cut":23,"wall_s":34} diff --git a/v13/z12.py b/v13/z12.py new file mode 100644 index 0000000000000000000000000000000000000000..030d9830c270646e3aa573f6ead43213a716089a --- /dev/null +++ b/v13/z12.py @@ -0,0 +1,5 @@ +import numpy as np +def z12(c): + s=lambda t,k:np.load(f'/workspace/huawei/v12/ens/{t}_c{c}_s0.npy')[:,k] + m=np.mean([np.load(f'/workspace/huawei/v12/ens/dwmlp_c{c}_s{k}.npy')[:,3] for k in (range(3) if c!=26 else [0])],0) + return ((s('dwbig',1)+s('dw_s',1)+2*m)/4).astype(np.float64) diff --git a/v14_transductive/README.md b/v14_transductive/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d160182cd5e4875f5332f96eaba4ee7596ff9615 --- /dev/null +++ b/v14_transductive/README.md @@ -0,0 +1,49 @@ +# v14 — TRANSDUCTIVE TEST VERSION: uses aggregates over unlabelled evaluation rows (normally banned by Eric; enabled only for this version) + +**Status: NO SUBMISSION BUILT. No transductive feature showed a gain beyond noise, so v14 is not worth submitting.** (`submission_v14_TRANSDUCTIVE.csv` was deliberately not produced; nothing was uploaded.) + +## Evaluation-row statistics that were tested (all label-free) +Code: `feat_tx.py::build_tx` (copy of /workspace/huawei/dw_lb/feat_tx.py). Target rows T = validation first-event rows (each user's first event at/after cutoff C, 10 candidates each; C=25 → Jan 25-27, C=23 → Jan 23-27). For a submission the same function would be run over the 52,939 evaluation.csv events (slates, ts, histories). n = number of target events. "j" = candidate item, i = the row's event. Own slot/own history is removed (leave-one-out). Train-side reference quantities use only train events with ts results_tx.jsonl +``` +(Copies of the scripts are in this folder; run from dw_lb because they import dw_levers/dw_signals read-only.) + +## Risks if someone did build it +- Window mismatch: validation windows are 2 days (C25) / 4 days (C23) but evaluation spans 7 days (Jan 27–Feb 3, 70% of events in the first 2 days), so count statistics would shift; they would have to be normalised per event (they are) and training folds would have to use matching windows. +- Public/private are random halves of evaluation.csv, so transductive statistics over ALL evaluation rows move both halves equally — no public-vs-private selection effect, but also no leak-driven protection against overfit. +- Training folds need mirrored statistics (per pseudo-cutoff fold over its own first-event rows); that rebuild is expensive and unjustified for ≤+0.0001. +- Violates the evaluation-aggregate ban unless explicitly accepted for this version. diff --git a/v14_transductive/feat_tx.py b/v14_transductive/feat_tx.py new file mode 100644 index 0000000000000000000000000000000000000000..872bc227100a94da62da3d92eb85677cd2cf5bf9 --- /dev/null +++ b/v14_transductive/feat_tx.py @@ -0,0 +1,115 @@ +"""Priority 5 (TRANSDUCTIVE, label-free): statistics over the unlabelled TARGET rows (validation first-event rows when validating; evaluation rows for v14). +Uses only: target slates (10 candidate items), target event ts, target histories (items+ts), and train-side exposure/CTR strictly before C (labels of ts slates, ts, histories (as unlabeled inputs)""" + S=d.RI[e]; ts=d.ETS[e]; hi=[d.HI[d.OFF[x]:d.OFF[x+1]] for x in e]; ht=[d.HT[d.OFF[x]:d.OFF[x+1]] for x in e] + return S,ts,hi,ht +def zrk(x): + m=x.reshape(-1,10); mu=np.nanmean(m,1,keepdims=True); sd=np.nanstd(m,1,keepdims=True)+1e-6 + return ((m-mu)/sd).ravel(),(m-mu).ravel(),m.argsort(1).argsort(1).astype(np.float32).ravel() +def build_tx(S,ts,hi,ht,C,ref,NI,fams=FAMS): + n=len(S); col={}; fl=S.ravel(); rep=lambda a:np.repeat(np.asarray(a,np.float32),10) + cnt=np.bincount(fl,minlength=NI).astype(np.float32) # exposure in target slates (includes own slot) + # ---- family 1: item exposure in target slates + if 'f1' in fams: + c1=cnt[fl]-1; col['f1_cnt']=c1/n*1e4 + o=np.argsort(ts,kind='stable'); tss=ts[o] + # time-windowed exposure: key=(item,ts) of every target slot + its=np.repeat(S[o].ravel(),1); kt=np.repeat(tss,10); key=np.sort(its.astype(np.int64)*(1<<33)+kt) + base=fl.astype(np.int64)*(1<<33); tr=np.repeat(ts,10).astype(np.int64) + for nm,w in (('h1',3600),('h6',6*3600),('h24',24*3600)): + lo=np.searchsorted(key,base+tr-w,'left'); hi_=np.searchsorted(key,base+tr+w,'right'); cw=(hi_-lo-1).astype(np.float32) + nev=(np.searchsorted(tss,ts+w,'right')-np.searchsorted(tss,ts-w,'left')).astype(np.float32) # #target events in the window + col['f1_cnt_'+nm]=cw/np.repeat(nev,10)*1e3 # per-event normalised local exposure share + if nm=='h6': col['f1_cnt_h6_raw']=cw + expl=ref['ex_l'][fl]/max(ref['nL'],1)*1e3; col['f1_late_exp']=expl; col['f1_late_ctr']=(ref['cl_l'][fl]+0.2)/(ref['ex_l'][fl]+2) + col['f1_ratio_late']=(c1/n+1e-4)/((ref['ex_l'][fl]+0.5)/max(ref['nL'],1)); col['f1_ratio_all']=(c1/n+1e-4)/((ref['ex_all'][fl]+0.5)/max(ref['nX'],1)) + col['f1_exp_x_ctr']=col['f1_cnt']*col['f1_late_ctr'] + for k in ('f1_cnt','f1_cnt_h6','f1_ratio_late'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + col['f1_slate_mean']=np.repeat(col['f1_cnt'].reshape(-1,10).mean(1),10) + # ---- family 3: popularity in the histories of target users (items clicked; incl. post-cutoff clicks visible in histories) + hlen=np.array([len(x) for x in hi]); hI=np.concatenate(hi).astype(np.int64) if hlen.sum() else np.zeros(0,np.int64); hT=np.concatenate(ht).astype(np.int64) + hu=np.repeat(np.arange(n),hlen); pos_from_end=np.concatenate([np.arange(l)[::-1] for l in hlen]) if hlen.sum() else np.zeros(0) + if 'f3' in fams: + hc=np.bincount(hI,minlength=NI).astype(np.float32); col['f3_hpop']=hc[fl]/n*1e4 + wl=np.bincount(hI,weights=(pos_from_end<3)/(1.0+pos_from_end),minlength=NI).astype(np.float32); col['f3_hpop_l3']=wl[fl]/n*1e4 + l1=np.bincount(hI[pos_from_end==0],minlength=NI).astype(np.float32); col['f3_hpop_l1']=l1[fl]/n*1e4 + post=hT>=C; hp=np.bincount(hI[post],minlength=NI).astype(np.float32); col['f3_hpost']=hp[fl]/n*1e4 + hk=np.sort(hI*(1<<33)+hT); base=fl.astype(np.int64)*(1<<33); tr=np.repeat(ts,10).astype(np.int64) + for nm,w in (('6h',6*3600),('24h',24*3600)): + lo=np.searchsorted(hk,base+tr-w,'left'); hi_=np.searchsorted(hk,base+tr,'left'); col['f3_hrecent_'+nm]=(hi_-lo).astype(np.float32) # others' clicks on j in the w hours before this event (visible in target histories) + lo2=np.searchsorted(hk,base+tr,'left'); hi2=np.searchsorted(hk,base+tr+w,'right'); col['f3_hnext_'+nm]=(hi2-lo2).astype(np.float32) + # own-history subtraction for the "post" count: own clicks of j in own history + own=np.zeros(n*10,np.float32) + for i in range(n): + if hlen[i]: + h=hi[i]; own[i*10:(i+1)*10]=(S[i][:,None]==h[None,:]).sum(1) + col['f3_own_cnt']=own; col['f3_hpop_excl_own']=col['f3_hpop']-own/n*1e4 + for k in ('f3_hpop','f3_hpost','f3_hrecent_24h'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + col['f3_hpost_vs_train_ctr']=col['f3_hpost']/(ref['cl_all'][fl]/max(ref['nX'],1)*1e4+1.0) + # ---- family 2: co-occurrence in target slates + last-click share among target users + if 'f2' in fams: + B=csr_matrix((np.ones(n*10,np.float32),(np.repeat(np.arange(n),10),fl)),shape=(n,NI)); CC=(B.T@B).tocsr() + ri=np.repeat(S,10,axis=1).ravel(); ci=np.tile(S,(1,10)).ravel() # all 100 pairs per event + v=np.asarray(CC[ri,ci]).ravel().reshape(n,10,10)-1.0 # LOO own + offd=~np.eye(10,dtype=bool); cj=(cnt[S]-1) # (n,10) + norm=np.sqrt((cj[:,:,None]+1)*(cj[:,None,:]+1)) + coo=(v/norm)[:,offd].reshape(n,10,9) if False else None + vm=np.where(offd[None],v/norm,0).sum(2)/9; col['f2_cooc_norm']=vm.ravel() + col['f2_cooc_raw']=(np.where(offd[None],v,0).sum(2)/9).ravel() + # last-click share: users whose last click was h -> share of slates containing j + last=np.array([x[-1] if len(x) else -1 for x in hi],np.int64); l3=[x[-3:] for x in hi] + H=csr_matrix((np.ones((last>=0).sum(),np.float32),(np.flatnonzero(last>=0),last[last>=0])),shape=(n,NI)); A=(H.T@B).tocsr(); nh=np.asarray(H.sum(0)).ravel() + lsh=np.full((n,10),np.nan,np.float32) + ok=np.flatnonzero(last>=0) + if len(ok): + a=np.asarray(A[np.repeat(last[ok],10),S[ok].ravel()]).ravel().reshape(-1,10)-1.0; den=np.maximum(nh[last[ok]]-1,0)[:,None] + lsh[ok]=np.where(den>0,a/np.maximum(den,1),np.nan) + col['f2_lastshare_n']=np.repeat(np.where(nh[last]>0,nh[last]-1,np.nan),10).astype(np.float32) + col['f2_lastshare']=lsh.ravel() + # any of last-3 clicks (weighted share) + H3=csr_matrix((np.concatenate([np.ones(len(x),np.float32) for x in l3]),(np.repeat(np.arange(n),[len(x) for x in l3]),np.concatenate(l3).astype(np.int64) if sum(len(x) for x in l3) else [])),shape=(n,NI)); A3=(H3.T@B).tocsr(); nh3=np.asarray(H3.sum(0)).ravel() + s3=np.zeros((n,10),np.float32); d3=np.zeros(n,np.float32) + for k in range(3): + idx=[i for i in range(n) if len(l3[i])>k] + if not idx: continue + idx=np.array(idx); h=np.array([l3[i][-1-k] for i in idx],np.int64) + a=np.asarray(A3[np.repeat(h,10),S[idx].ravel()]).ravel().reshape(-1,10)-1.0 + s3[idx]+=a; d3[idx]+=np.maximum(nh3[h]-1,0) + col['f2_l3share']=np.where(d3[:,None]>0,s3/np.maximum(d3[:,None],1),np.nan).ravel() + for k in ('f2_cooc_norm','f2_lastshare','f2_l3share'): + z,dm,rk=zrk(col[k]); col[k+'_z']=z; col[k+'_dm']=dm; col[k+'_rk']=rk + # ---- family 4: timing among target events + if 'f4' in fams: + o=np.argsort(ts,kind='stable'); rk_=np.empty(n); rk_[o]=np.arange(n); tss=ts[o] + col['f4_trel_h']=rep((ts-C)/3600); col['f4_trank']=rep(rk_/n) + col['f4_n_same_h']=rep(np.searchsorted(tss,ts+1800,'right')-np.searchsorted(tss,ts-1800,'left')) + col['f4_n_6h']=rep(np.searchsorted(tss,ts+6*3600,'right')-np.searchsorted(tss,ts-6*3600,'left')) + col['f4_n_before_frac']=rep(rk_/n) + col['f4_hist_n']=rep(hlen) + names=list(col); X=np.column_stack([np.asarray(col[k],np.float32) for k in names]).astype(np.float32) + return X,names +def build_val(d,c,fams=FAMS): + C=Jd(c); v=d.val(c); e=v['e'][::10]; S,ts,hi,ht=target_from_data(d,e) + # sanity: slate order must match val row order + S=v['item'].reshape(-1,10).astype(np.int64) # slate in val-row order + assert (np.sort(S,1)==np.sort(d.RI[e],1)).all() + return build_tx(S,ts,hi,ht,C,train_ref(d,C),d.NI,fams) +if __name__=='__main__': + d=D.Data() + for c in [int(x) for x in sys.argv[1].split(',')]: + X,names=build_val(d,c); np.save(f'/workspace/huawei/dw_lb/cache_tx_c{c}.npy',X); json.dump(names,open(f'/workspace/huawei/dw_lb/tx_names.json','w')); print(c,X.shape,flush=True) diff --git a/v14_transductive/lib.py b/v14_transductive/lib.py new file mode 100644 index 0000000000000000000000000000000000000000..fd28528ba6fb8541fd41fb5164356e7fc6d8ea5e --- /dev/null +++ b/v14_transductive/lib.py @@ -0,0 +1,43 @@ +import sys, os, json, numpy as np, polars as pl, lightgbm as lgb, warnings; warnings.filterwarnings('ignore') +sys.dont_write_bytecode=True +sys.path.insert(0,'/workspace/huawei/dw_levers'); sys.path.insert(1,'/workspace/huawei/dw_signals') +from sklearn.metrics import roc_auc_score as A +from sklearn.model_selection import GroupKFold +H='/workspace/huawei'; DAY=86400; J20=1737331200 +def Jd(d): return J20+(d-20)*DAY +def z12(c): + s=lambda t,k:np.load(f'{H}/v12/ens/{t}_c{c}_s0.npy')[:,k] + m=np.mean([np.load(f'{H}/v12/ens/dwmlp_c{c}_s{k}.npy')[:,3] for k in (range(3) if c!=26 else [0])],0) + return ((s('dwbig',1)+s('dw_s',1)+2*m)/4).astype(np.float64) +def meta(c): return pl.read_parquet(f'{H}/v10/feat_v3/val{c}.parquet',columns=['rowid','eid','u','ts','clicked']) +def base_ctx(z): + Z=z.reshape(-1,10); mu=Z.mean(1,keepdims=True); sd=Z.std(1,keepdims=True)+1e-6 + rk=Z.argsort(1).argsort(1).astype(np.float32) + lse=np.log(np.exp(Z).sum(1,keepdims=True)) + cols=[z,((Z-mu)/sd).ravel(),(Z-mu).ravel(),rk.ravel(),np.repeat(lse,10,1).ravel(),np.repeat(Z.max(1,keepdims=True),10,1).ravel(),np.repeat(mu,10,1).ravel(),np.repeat(sd,10,1).ravel()] + return np.column_stack(cols).astype(np.float32) +PI=dict(objective='binary',learning_rate=0.03,num_leaves=15,min_data_in_leaf=500,feature_fraction=0.8,bagging_fraction=0.8,bagging_freq=1,lambda_l2=30,max_bin=127,verbose=-1,num_threads=4) +def oof(X,y,z,user,seed=0,rounds=200,k=5,params=None): + p=np.zeros(len(y)); P={**PI,**(params or {})} + for tr,te in GroupKFold(k).split(X,y,user): + m=lgb.train({**P,'seed':seed},lgb.Dataset(X[tr],y[tr],init_score=z[tr]),rounds); p[te]=m.predict(X[te],raw_score=True)+z[te] + return p +def boot_delta(y,p0,p1,ev,nb=200,seed=0): + """bootstrap over events (10 rows each) of AUC(p1)-AUC(p0)""" + rs=np.random.RandomState(seed); n=len(y)//10; Y=y.reshape(n,10); a=p0.reshape(n,10); b=p1.reshape(n,10); out=[] + for _ in range(nb): + i=rs.randint(0,n,n); yy=Y[i].ravel(); out.append(A(yy,b[i].ravel())-A(yy,a[i].ravel())) + return np.percentile(out,[2.5,97.5]), np.std(out) +def load_cut(c): + m=meta(c); z=z12(c); assert len(z)==m.height + return dict(c=c,y=m['clicked'].to_numpy().astype(int),z=z,user=m['u'].to_numpy(),rowid=m['rowid'].to_numpy(),ts=m['ts'].to_numpy(),Xb=base_ctx(z)) +def report(name,F,Xnew,seeds=(0,),log='results.jsonl',control_cols=None): + """F: cut dict; Xnew: (n,k) new feats. control = Xb only; test = Xb+Xnew. returns delta vs z and vs control""" + y,z,u=F['y'],F['z'],F['user']; res=dict(name=name,cut=F['c'],nfeat=Xnew.shape[1],base=A(y,z)) + pc=np.mean([oof(F['Xb'],y,z,u,s) for s in seeds],0); X=np.hstack([F['Xb'],Xnew]); pt=np.mean([oof(X,y,z,u,s) for s in seeds],0) + res.update(control=A(y,pc),test=A(y,pt)); res['d_vs_z']=res['test']-res['base']; res['d_vs_ctrl']=res['test']-res['control'] + ci,sd=boot_delta(y,pc,pt,None); res['ci_vs_ctrl']=[float(ci[0]),float(ci[1])]; res['sd']=float(sd) + ev=lambda p: A((y.reshape(-1,10).max(1)),np.log(np.exp(p.reshape(-1,10)).sum(1))) + res['evAUC_base']=ev(z); res['evAUC_test']=ev(pt) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in res.items()}),flush=True) + open(f'/workspace/huawei/dw_lb/{log}','a').write(json.dumps(res)+'\n'); return res,pt diff --git a/v14_transductive/results_tx.jsonl b/v14_transductive/results_tx.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b347b3cdc19cd4cd3e5f39cc95b902a6419ed971 --- /dev/null +++ b/v14_transductive/results_tx.jsonl @@ -0,0 +1,18 @@ +{"name": "tx_f1", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172027883512023, "d_vs_ctrl": -1.6709914137047477e-05, "ci": [-0.00014349266344125257, 0.00014587432118149673], "sd": 7.394595909960953e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173499925039057, "d_vs_ctrl": 0.00013049423856636988, "ci": [3.9790617533291826e-05, 0.00023067132830789097], "sd": 5.150322756776621e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173169526524945, "d_vs_ctrl": 9.74543871551381e-05, "ci": [-5.4981787659713624e-05, 0.00021526865679996352], "sd": 7.300359621086734e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8172527064819, "d_vs_ctrl": 3.320821656060957e-05, "ci": [-5.4703244326295694e-05, 0.00011614454119170672], "sd": 4.313636553203988e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173167974531333, "d_vs_ctrl": 9.729918779399771e-05, "ci": [-7.801107898108007e-05, 0.00023711497566265008], "sd": 8.903256137410697e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 0, "base": 0.8172259475835104, "control": 0.8172194982653394, "test": 0.8173983206258677, "d_vs_ctrl": 0.00017882236052835232, "ci": [-5.831556720231612e-06, 0.00032584640888283224], "sd": 8.898557768981401e-05, "nfeat": 88} +{"name": "tx_f1", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956233103415669, "d_vs_ctrl": 7.787187815710972e-05, "ci": [-7.479367148836014e-05, 0.00026347507973854004], "sd": 8.39083487483491e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7954944826046069, "d_vs_ctrl": -5.095585880288134e-05, "ci": [-0.00013868587711693414, 4.4620645681803275e-05], "sd": 4.891152042204878e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956304922619755, "d_vs_ctrl": 8.505379856571516e-05, "ci": [-4.9795107496017343e-05, 0.00022870341233730828], "sd": 7.476699867419253e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956183987133403, "d_vs_ctrl": 7.296024993053152e-05, "ci": [-3.4362482472535814e-05, 0.0001534660315574921], "sd": 4.601110414256159e-05, "nfeat": 6} +{"name": "tx_all", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7956668950159012, "d_vs_ctrl": 0.00012145655249140574, "ci": [-4.162412313422625e-05, 0.0002827905883446003], "sd": 8.632411276152597e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 23, "seed": 0, "base": 0.7956557773874656, "control": 0.7955454384634097, "test": 0.7958353874293667, "d_vs_ctrl": 0.0002899489659569099, "ci": [0.00012652061304543327, 0.00046809093858551367], "sd": 8.882145961222389e-05, "nfeat": 88} +{"name": "tx_f1", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8171540011704558, "d_vs_ctrl": -4.7271458446407344e-05, "ci": [-0.00019061083887653984, 9.264123771784986e-05], "sd": 8.080654707256493e-05, "nfeat": 20} +{"name": "tx_f2", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173168932756604, "d_vs_ctrl": 0.00011562064675818551, "ci": [3.137035239061392e-05, 0.0002165138368945774], "sd": 4.745692855025363e-05, "nfeat": 14} +{"name": "tx_f3", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173267896762714, "d_vs_ctrl": 0.00012551704736918357, "ci": [-1.1038759591067149e-05, 0.0002785910167285549], "sd": 7.539173816284816e-05, "nfeat": 20} +{"name": "tx_f4", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8172492119677455, "d_vs_ctrl": 4.793933884328361e-05, "ci": [-4.7446992155203425e-05, 0.00013619981497832174], "sd": 4.878073514728382e-05, "nfeat": 6} +{"name": "tx_all", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.817286679273291, "d_vs_ctrl": 8.540664438883017e-05, "ci": [-9.598281369740669e-05, 0.00025745601599898863], "sd": 8.95598780600043e-05, "nfeat": 60} +{"name": "tx_all+prop", "cut": 25, "seed": 1, "base": 0.8172259475835104, "control": 0.8172012726289022, "test": 0.8173714409684247, "d_vs_ctrl": 0.00017016833952254729, "ci": [9.941956942391419e-06, 0.00032593622615555433], "sd": 8.941576662609524e-05, "nfeat": 88} diff --git a/v14_transductive/run_tx.py b/v14_transductive/run_tx.py new file mode 100644 index 0000000000000000000000000000000000000000..5913f3e99ba0fbb0fb4e59e527302a8c7d110d1e --- /dev/null +++ b/v14_transductive/run_tx.py @@ -0,0 +1,14 @@ +from lib import * +import sys +cuts=[int(x) for x in sys.argv[1].split(',')]; seeds=[int(x) for x in sys.argv[2].split(',')] +names=json.load(open('tx_names.json')); fam={f:[i for i,n in enumerate(names) if n.startswith(f)] for f in ['f1','f2','f3','f4']} +sets=dict(f1=fam['f1'],f2=fam['f2'],f3=fam['f3'],f4=fam['f4'],all=list(range(len(names)))) +for c in cuts: + F=load_cut(c); T=np.load(f'cache_tx_c{c}.npy'); P=np.load(f'cache_prop_c{c}.npy'); y,z,u=F['y'],F['z'],F['user'] + for s in seeds: + pc=oof(F['Xb'],y,z,u,s); + for nm,idx in list(sets.items())+[('all+prop',None)]: + X=np.hstack([F['Xb'],T if idx is None else T[:,idx]]+([P] if idx is None else [])) + pt=oof(X,y,z,u,s); ci,sd=boot_delta(y,pc,pt,None,nb=150) + r=dict(name='tx_'+nm,cut=c,seed=s,base=A(y,z),control=A(y,pc),test=A(y,pt),d_vs_ctrl=A(y,pt)-A(y,pc),ci=[float(ci[0]),float(ci[1])],sd=float(sd),nfeat=X.shape[1]-F['Xb'].shape[1]) + print(json.dumps({k:(round(v,5) if isinstance(v,float) else v) for k,v in r.items()}),flush=True); open('results_tx.jsonl','a').write(json.dumps(r)+'\n')