sync 2026-10-23 13:31 IST | best val AUC 0.777500

README.md

@@ -10,7 +10,7 @@ tags: [ctr, lightgbm, tabular, recsys]
 
 **No competition data is included** — no train/evaluation files, feature caches, parquet/duckdb, or row-level predictions/submissions. Only code, logs, metrics, charts and LightGBM model dumps.
 
-_Last sync: 2026-10-23 12:26 IST · 165 files._
+_Last sync: 2026-10-23 13:31 IST · 290 files._
 
 ## Submission log (`v2work/out/submission_log.csv`)
 
@@ -19,7 +19,7 @@ date,file,sha256,val_auc,public_auc
 2026-10-23,submission_v2.csv,8a868ba5edac1b7fed695479cff9d2553f28d5cfa797805192d26a352930d1b4,0.7620,
 2026-10-23,submission_v3.csv,466f3d64b8b273ac1c87c32a8d271761add9a925cb1f56979945a5d004cd2680,0.7775,
 2026-10-23,submission_v2.csv,8a868ba5edac1b7fed695479cff9d2553f28d5cfa797805192d26a352930d1b4,SUPERSEDED (eval aggregates: pooled eval-row histories into item stats/cooc/SVD) - do not upload,
-2026-10-23,submission_v3.csv.zip,public_auc,,0.788757
+2026-10-23,submission_v3.csv.zip (public score),466f3d64b8b273ac1c87c32a8d271761add9a925cb1f56979945a5d004cd2680,0.7775,0.788757
 ```
 
 ## v3 validation runs (`v3/out/results.jsonl`)

parallel/lag/b_v2.py

@@ -0,0 +1,7 @@
+import sys; sys.path.insert(0,'/workspace/huawei/parallel/src_snapshot/v2work')
+from feats import load, first_events, build, log
+D='/workspace/huawei/v2work'; DAY=86400; J20=1737331200
+TE,TR,EE,ER,NI=load(D)
+T_ev=first_events(TE,J20+5*DAY,2*DAY); T_rows=TR.join(T_ev.select('eid'),on='eid')
+R=build(J20+4*DAY,T_ev,T_rows,TE,TR,NI)
+R.drop('h','h_ts').write_parquet('/workspace/huawei/parallel/lag/feat/val25lag.parquet'); log('wrote',R.shape)

parallel/lag/b_v3.py

@@ -0,0 +1,8 @@
+"""lag-1d: val25 targets (first event/user >= Jan25) with ALL stats frozen at Jan24 (v3feat_ne.build from snapshot)."""
+import sys; sys.path.insert(0,'/workspace/huawei/parallel/src_snapshot/v2work'); sys.path.insert(1,'/workspace/huawei/v3')
+from v3feat_ne import load, build, log
+from split import J20, DAY, first_events
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+T_ev = first_events(TE, J20 + 5 * DAY)
+R = build(J20 + 4 * DAY, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU)
+R.write_parquet('/workspace/huawei/parallel/lag/feat_v3/val25lag.parquet'); log('wrote', R.shape)

parallel/lag/build.log

@@ -0,0 +1,9 @@
+12:37:57 build C=1737676800 rows=439640 cols=170 38s
+12:37:57 wrote (439640, 170)
+12:38:02 pool 880430
+12:38:05 cooc pairs 15164046
+12:38:06 svd done
+12:38:28 build C=1737676800 rows=439640 28s
+12:38:29 wrote (439640, 125)
+val25lag (439640, 39) 13s
+val25lag (439640, 20) 33s

parallel/lag/cmp.py

@@ -0,0 +1,14 @@
+import polars as pl, numpy as np, json
+V='/workspace/huawei/v2work'; L='/workspace/huawei/parallel/lag'; out={}
+for d,a,b in (('feat_v3','val25','val25repro'),('feat','val25','val25repro'),('feat_x','val25','val25'),('feat_x2','val25','val25')):
+    A=pl.read_parquet(f'{V}/{d}/{a}.parquet'); B=pl.read_parquet(f'{L}/{d}/{b}.parquet')
+    B=A.select('rowid').join(B,on='rowid',how='left')
+    cols=[c for c in A.columns if c in B.columns and c!='rowid' and A[c].dtype.is_numeric()]
+    bad={}
+    for c in cols:
+        x=A[c].cast(pl.Float64).to_numpy(); y=B[c].cast(pl.Float64).to_numpy()
+        dif=np.nanmax(np.abs(x-y)) if np.isfinite(x-y).any() else 0.0
+        nanm=int((np.isnan(x)!=np.isnan(y)).sum())
+        if dif>1e-3*max(1,np.nanmax(np.abs(x))) or nanm: bad[c]=(float(dif),nanm)
+    out[d]=dict(ncols=len(cols),only_disk=[c for c in A.columns if c not in B.columns],n_diff=len(bad),diff=dict(list(bad.items())[:12]))
+print(json.dumps(out,indent=1)); json.dump(out,open(f'{L}/repro_cmp.json','w'),indent=1)

parallel/lag/cmp2.py

@@ -0,0 +1,14 @@
+import polars as pl, numpy as np, json
+V='/workspace/huawei/v2work'; L='/workspace/huawei/parallel/lag'; d0=json.load(open(f'{L}/repro_cmp.json')); out={}
+for d,b in (('feat_v3','val25repro'),('feat','val25repro'),('feat_x','val25')):
+    A=pl.read_parquet(f'{V}/{d}/val25.parquet'); B=A.select('rowid').join(pl.read_parquet(f'{L}/{d}/{b}.parquet'),on='rowid',how='left')
+    full=json.load(open(f'{L}/repro_cmp.json'))[d]
+    import itertools
+    cols=[c for c in A.columns if c!='rowid' and c in B.columns and A[c].dtype.is_numeric()]
+    r={}
+    for c in cols:
+        x=A[c].cast(pl.Float64).to_numpy(); y=B[c].cast(pl.Float64).to_numpy(); m=np.isfinite(x)&np.isfinite(y)
+        if np.allclose(x[m],y[m],rtol=1e-4,atol=1e-5): continue
+        r[c]=dict(corr=round(float(np.corrcoef(x[m],y[m])[0,1]),5),frac_rows_diff=round(float((~np.isclose(x[m],y[m],rtol=1e-3,atol=1e-4)).mean()),4))
+    out[d]=dict(n_cols=len(cols),n_cols_diff=len(r),worst=sorted(r.items(),key=lambda kv:kv[1]['corr'])[:8])
+print(json.dumps(out,indent=1)); json.dump(out,open(f'{L}/repro_cmp2.json','w'),indent=1)

parallel/lag/extra.py

@@ -0,0 +1,79 @@
+"""Extra leak-free features on v3 data (all stats from data with ts < C).
+Usage: python extra.py NAME...   (names: f1..f6, val25, val26, eval)  -> v2work/feat_x/NAME.parquet
+ - con variants: k=32/128, recency-weighted (tau=2d) click/non-click centroids, item-side con (users who clicked i vs skipped i)
+ - user-cluster item CTR (kmeans on user SVD vectors, 64 & 256 clusters), smoothed toward item CTR
+ - within-event z-score and rank for each"""
+import sys, os, time, numpy as np, polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+from sklearn.cluster import MiniBatchKMeans
+sys.path.insert(0, '/workspace/huawei/v3')
+from features import load, click_log
+from split import J20, DAY, first_events
+OUT = '/workspace/huawei/parallel/lag/feat_x'; os.makedirs(OUT, exist_ok=True)
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'),
+        **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}}
+def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
+def csr(u, i, w): return sparse.csr_matrix((w.astype(np.float32), (u, i)), shape=(NU, NI))
+def build(C, T_ev, T_rows):
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    CL = click_log(C, P_ev, P_rows, T_ev)            # user-level: train data + each target row's own history
+    CLg = click_log(C, P_ev, P_rows, T_ev.head(0))   # global/item-level: train data only
+    assert CL['ts'].max() < C and P_rows['ts'].max() < C
+    R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii')
+    uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+    cg = CLg.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = csr(cg['u'].to_numpy(), cg['ii'].to_numpy(), np.ones(cg.height)); Scg = Xc.copy()
+    cnt = CL.group_by('u', 'ii').agg(pl.len().alias('n'), pl.col('ts').max().alias('lt'))
+    cu = cnt['u'].to_numpy(); ci = cnt['ii'].to_numpy()
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc
+    Xu = csr(cu, ci, np.ones(cnt.height)) @ sparse.diags(idf); Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu
+    neg = P_rows.filter(pl.col('clicked') == 0)
+    nu_ = neg['u'].to_numpy(); ni_ = neg['ii'].to_numpy()
+    Sc = csr(cu, ci, np.ones(cnt.height)); Sn = csr(nu_, ni_, np.ones(neg.height))
+    rc = np.exp(-(C - cnt['lt'].to_numpy()) / (2 * DAY)); rn = np.exp(-(C - neg['ts'].to_numpy()) / (2 * DAY))
+    Scr = csr(cu, ci, rc); Snr = csr(nu_, ni_, rn)
+    F = {}
+    for k in (32, 64, 128):
+        U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)
+        Vn = nrm(Vt.T * np.sqrt(S)); Un = nrm(np.asarray(Xu @ (Vt.T / np.sqrt(S)))); cand = Vn[ii]  # user fold-in
+        nC = nrm(np.asarray(Sc @ Vn)); nN = nrm(np.asarray(Sn @ Vn))
+        F[f'x_con05_k{k}'] = (nrm(nC - 0.5 * nN)[uu] * cand).sum(1)
+        if k == 64:
+            nCr = nrm(np.asarray(Scr @ Vn)); nNr = nrm(np.asarray(Snr @ Vn))
+            for a, nm in ((0.5, '05'), (1.0, '1')):
+                F[f'x_conr{nm}_k64'] = (nrm(nCr - a * nNr)[uu] * cand).sum(1)
+            F['x_crcos_k64'] = (nCr[uu] * cand).sum(1)
+            # item side: centroid of users who clicked item minus users who skipped it, vs target user vector
+            iC = nrm(np.asarray(Scg.T @ Un)); iN = nrm(np.asarray(Sn.T @ Un))  # item side: train-only clicks
+            F['x_icon05_k64'] = (nrm(iC - 0.5 * iN)[ii] * Un[uu]).sum(1)
+            F['x_icos_k64'] = (iC[ii] * Un[uu]).sum(1)
+            # user-cluster CTR
+            act = np.unique(np.concatenate([P_rows['u'].to_numpy(), cg['u'].to_numpy()]))
+            for nc in (64, 256):
+                km = MiniBatchKMeans(nc, random_state=0, batch_size=4096, n_init=3).fit(Un[act])
+                lab = km.predict(Un).astype(np.int32)
+                g = float(P_rows['clicked'].mean())
+                pr = P_rows.select('ii', 'clicked', pl.Series('cl', lab[P_rows['u'].to_numpy()]))
+                ict = pr.group_by('ii').agg(((pl.col('clicked').sum() + 20 * g) / (pl.len() + 20)).alias('ictr'))
+                cct = pr.group_by('cl', 'ii').agg(pl.col('clicked').sum().alias('c'), pl.len().alias('n'))
+                q = pl.DataFrame({'cl': lab[uu], 'ii': ii.astype(np.int32), 'r': np.arange(len(uu))}).join(cct, on=['cl', 'ii'], how='left') \
+                      .join(ict, on='ii', how='left').sort('r').with_columns(pl.col('ictr').fill_null(g), pl.col('c').fill_null(0), pl.col('n').fill_null(0))
+                F[f'x_clctr{nc}'] = ((q['c'] + 10 * q['ictr']) / (q['n'] + 10)).to_numpy()
+                F[f'x_clctr{nc}_lift'] = F[f'x_clctr{nc}'] / q['ictr'].to_numpy()
+                F[f'x_climp{nc}'] = q['n'].to_numpy().astype(np.float32)
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])
+    zc = [c for c in F if not c.startswith('x_climp')]
+    R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in zc] +
+                       [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in zc])
+    return R.drop('eid', 'u', 'ii')
+for name in sys.argv[1:]:
+    t0 = time.time(); C, W = (J20 + 4 * DAY, 'lag') if name == 'val25lag' else SPEC[name]
+    if W == 'lag': T_ev = first_events(TE, J20 + 5 * DAY, None); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    elif W == 'eval': R = build(C, EE, ER)
+    else:
+        T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True)

parallel/lag/extra2.py

@@ -0,0 +1,74 @@
+"""Extra features #2 (leak-free; everything from ts < C):
+ - ridge 'taste vector' per user on frozen SVD k64 item vectors (clicks y=1 incl. pre-C click log, skipped impressions y=0 or -0.5)
+ - user state: time since last event/click, event-level click rate, recent-event click streaks, events in last 24h before C
+Usage: python extra2.py NAME... -> v2work/feat_x2/NAME.parquet"""
+import sys, os, time, numpy as np, polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+sys.path.insert(0, '/workspace/huawei/v3')
+from features import load, click_log
+from split import J20, DAY, first_events
+OUT = '/workspace/huawei/parallel/lag/feat_x2'; os.makedirs(OUT, exist_ok=True)
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None),
+        'val22': (J20 + 2 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'),
+        **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}}
+def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
+def ridge_users(Wm, Ym, V, lam):
+    """per-user solve (V_I^T W V_I + lam I) w = V_I^T (W*y). Wm, Ym: csr users x items (weights, weight*target)."""
+    k = V.shape[1]; A = np.zeros((Wm.shape[0], k, k), np.float32)
+    for a in range(k):
+        cols = Wm @ (V * V[:, a:a + 1])          # (NU, k): sum_i w_ui v_ia v_ib
+        A[:, a, :] = cols
+    A += lam * np.eye(k, dtype=np.float32)[None]
+    b = np.asarray(Ym @ V, dtype=np.float32)
+    return np.linalg.solve(A, b[:, :, None])[:, :, 0]
+def build(C, T_ev, T_rows):
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    CL = click_log(C, P_ev, P_rows, T_ev)            # user-level: train data + each target row's own history
+    CLg = click_log(C, P_ev, P_rows, T_ev.head(0))   # global/item-level: train data only
+    assert CL['ts'].max() < C and P_rows['ts'].max() < C
+    R = T_rows.join(T_ev.select('eid', 'u', 'ts'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii', 'ts')
+    uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+    # SVD exactly as v3 (tf-idf, row-normalised user x item click matrix)
+    cnt = CLg.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc
+    U, S, Vt = randomized_svd(Xc, 64, random_state=0, n_iter=5); Vn = nrm(Vt.T * np.sqrt(S)).astype(np.float32)
+    # (user,item) labels: clicked anywhere before C -> 1 ; shown-but-never-clicked -> 0
+    pos = CL.select('u', 'ii').unique().with_columns(pl.lit(1.0).alias('y'))
+    neg = P_rows.filter(pl.col('clicked') == 0).select('u', 'ii').unique().join(pos.select('u', 'ii'), on=['u', 'ii'], how='anti').with_columns(pl.lit(0.0).alias('y'))
+    L = pl.concat([pos, neg])
+    lu = L['u'].to_numpy(); li = L['ii'].to_numpy(); ly = L['y'].to_numpy().astype(np.float32)
+    Wm = sparse.csr_matrix((np.ones(len(ly), np.float32), (lu, li)), shape=(NU, NI))
+    cand = Vn[ii]; F = {}
+    for lam, yneg, nm in ((1.0, 0.0, 'l1'), (10.0, 0.0, 'l10'), (3.0, -0.5, 'l3n')):
+        yy = np.where(ly > 0, 1.0, yneg).astype(np.float32)
+        Ym = sparse.csr_matrix((yy, (lu, li)), shape=(NU, NI))
+        Wu = ridge_users(Wm, Ym, Vn, lam)
+        F[f'r_{nm}'] = (Wu[uu] * cand).sum(1)
+    # user state features
+    ue = P_ev.select('eid', 'u', 'ts').join(P_rows.group_by('eid').agg(pl.col('clicked').sum().alias('nc')), on='eid', how='left') \
+             .with_columns(pl.col('nc').fill_null(0)).sort('u', 'ts')
+    us = ue.group_by('u').agg(pl.col('ts').last().alias('last_ev'), (pl.col('nc') > 0).mean().alias('s_evrate'), pl.len().alias('s_nev'),
+                              pl.col('nc').tail(1).sum().alias('s_clk_last1'), pl.col('nc').tail(3).sum().alias('s_clk_last3'),
+                              (pl.col('nc').tail(5) > 0).sum().alias('s_evclk_last5'), (pl.col('ts') >= C - DAY).sum().alias('s_nev24h'),
+                              pl.col('ts').filter(pl.col('nc') > 0).max().alias('last_clkev'))
+    lc = CL.group_by('u').agg(pl.col('ts').max().alias('last_click'), (pl.col('ts') >= C - DAY).sum().alias('s_clk24h'))
+    E = T_ev.select('eid', 'u', 'ts').join(us, on='u', how='left').join(lc, on='u', how='left').with_columns(
+        ((pl.col('ts') - pl.col('last_ev')) / 3600).alias('s_since_ev_h'), ((pl.col('ts') - pl.col('last_click')) / 3600).alias('s_since_clk_h'),
+        ((pl.col('ts') - pl.col('last_clkev')) / 3600).alias('s_since_clkev_h')).drop('u', 'ts', 'last_ev', 'last_click', 'last_clkev')
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])
+    R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in F] +
+                       [(pl.col(c) - pl.col(c).max().over('eid')).alias(c + '_dmax') for c in F])
+    R = R.join(E, on='eid', how='left')
+    return R.drop('eid', 'u', 'ii', 'ts')
+for name in sys.argv[1:]:
+    t0 = time.time(); C, W = (J20 + 4 * DAY, 'lag') if name == 'val25lag' else SPEC[name]
+    if W == 'lag': T_ev = first_events(TE, J20 + 5 * DAY, None); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    elif W == 'eval': R = build(C, EE, ER)
+    else:
+        T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True)

parallel/lag/lag_buckets.json

@@ -0,0 +1 @@
+{"val25": {"auc": 0.77748, "le1d": 0.78184, "d1_2": 0.75895, "event": 0.65358}, "val25lag": {"auc": 0.7647, "le1d": 0.76946, "d1_2": 0.7443, "event": 0.64837}}

parallel/lag/lagtest.json

@@ -0,0 +1,13 @@
+{
+ "n_feats": 325,
+ "val25": {
+  "200": 0.77684,
+  "300": 0.77724,
+  "400": 0.77748
+ },
+ "val25lag": {
+  "200": 0.76434,
+  "300": 0.76472,
+  "400": 0.7647
+ }
+}

parallel/lag/lagtest.py

@@ -0,0 +1,37 @@
+"""Train v2work's combo config (notime, XF=1, XF2=1, f1..f4, PARAMS from snapshot combo.py, 400 rounds, 3 threads) on v2work's
+on-disk tables (read-only). Then predict (a) val25 as built and (b) val25lag (the same target rows, all stats frozen at Jan24).
+Preds are saved in parallel/lag/."""
+import sys, os, json, gc, numpy as np, polars as pl, lightgbm as lgb
+sys.path.insert(0,'/workspace/huawei/parallel/src_snapshot/v2work'); sys.path.insert(1,'/workspace/huawei/v3')
+from feats import DROP
+from features import feature_cols, GROUPS
+from sklearn.metrics import roc_auc_score
+V='/workspace/huawei/v2work'; L='/workspace/huawei/parallel/lag'
+V2_OF={'f1':'h1','f2':'h2','f3':'h3','f4':'h4','val25':'val25','val25lag':'val25lag'}
+TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour']
+def load(n, base):
+    a=pl.read_parquet(f'{base}/feat_v3/{n}.parquet'); b=pl.read_parquet(f'{base}/feat/{V2_OF[n]}.parquet')
+    f2=[c for c in b.columns if c not in DROP]; b=b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2})
+    a=a.join(b,on='rowid',how='left')
+    a=a.join(pl.read_parquet(f'{base}/feat_x/{n}.parquet'),on='rowid',how='left')
+    return a.join(pl.read_parquet(f'{base}/feat_x2/{n}.parquet'),on='rowid',how='left')
+PARAMS=dict(objective='binary',learning_rate=0.05,num_leaves=63,min_data_in_leaf=200,feature_fraction=0.7,bagging_fraction=0.8,
+            bagging_freq=1,lambda_l2=10.0,verbose=-1,seed=42,num_threads=3,deterministic=True)
+feats=None; Xs=[]; ys=[]
+for k in range(1,5):
+    t=load(f'f{k}',V)
+    if feats is None:
+        feats=[c for c in feature_cols(t,GROUPS)+[c for c in t.columns if c.startswith(('v2_','x_','r_','s_'))] if c not in TIME]
+    Xs.append(t.select(feats).to_numpy().astype(np.float32)); ys.append(t['clicked'].to_numpy()); del t; gc.collect()
+X=np.concatenate(Xs); del Xs; gc.collect()
+print('train',X.shape,flush=True)
+ds=lgb.Dataset(X,np.concatenate(ys),feature_name=feats,free_raw_data=True).construct(); del X; gc.collect()
+m=lgb.train(PARAMS,ds,num_boost_round=400); del ds; gc.collect(); m.save_model(f'{L}/model_f1to4.txt')
+res={'n_feats':len(feats)}
+for n,base in (('val25',V),('val25lag',L)):
+    va=load(n,base); miss=[c for c in feats if c not in va.columns]; assert not miss, miss
+    Xv=va.select(feats).to_numpy().astype(np.float32); y=va['clicked'].to_numpy()
+    res[n]={k:round(roc_auc_score(y,m.predict(Xv,num_iteration=k)),5) for k in (200,300,400)}
+    pl.DataFrame({'rowid':va['rowid'],'p':m.predict(Xv,num_iteration=400)}).write_parquet(f'{L}/pred_{n}.parquet')
+    print(n,res[n],flush=True)
+json.dump(res,open(f'{L}/lagtest.json','w'),indent=1)

parallel/lag/r_v2.py

@@ -0,0 +1,7 @@
+import sys; sys.path.insert(0,'/workspace/huawei/parallel/src_snapshot/v2work')
+from feats import load, first_events, build, log
+D='/workspace/huawei/v2work'; DAY=86400; J20=1737331200
+TE,TR,EE,ER,NI=load(D)
+T_ev=first_events(TE,J20+5*DAY,2*DAY); T_rows=TR.join(T_ev.select('eid'),on='eid')
+R=build(J20+5*DAY,T_ev,T_rows,TE,TR,NI)
+R.drop('h','h_ts').write_parquet('/workspace/huawei/parallel/lag/feat/val25repro.parquet'); log('wrote',R.shape)

parallel/lag/r_v3.py

@@ -0,0 +1,8 @@
+"""lag-1d: val25 targets (first event/user >= Jan25) with ALL stats frozen at Jan24 (v3feat_ne.build from snapshot)."""
+import sys; sys.path.insert(0,'/workspace/huawei/parallel/src_snapshot/v2work'); sys.path.insert(1,'/workspace/huawei/v3')
+from v3feat_ne import load, build, log
+from split import J20, DAY, first_events
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+T_ev = first_events(TE, J20 + 5 * DAY)
+R = build(J20 + 5 * DAY, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU)
+R.write_parquet('/workspace/huawei/parallel/lag/feat_v3/val25repro.parquet'); log('wrote', R.shape)

parallel/lag/repro_cmp.json

@@ -0,0 +1,173 @@
+{
+ "feat_v3": {
+  "ncols": 169,
+  "only_disk": [],
+  "n_diff": 35,
+  "diff": {
+   "co_cos_max20": [
+    0.07715167850255966,
+    0
+   ],
+   "co_cos_mean20": [
+    0.018184823915362358,
+    0
+   ],
+   "co_cos_max5": [
+    0.0690065547823906,
+    0
+   ],
+   "co_cos_mean5": [
+    0.027216553688049316,
+    0
+   ],
+   "co_cos_last": [
+    0.06804138422012329,
+    0
+   ],
+   "co_tr_max20": [
+    0.0025684931315481663,
+    0
+   ],
+   "co_tr_mean20": [
+    0.0021482277661561966,
+    0
+   ],
+   "co_tr_max5": [
+    0.0025641026441007853,
+    0
+   ],
+   "co_tr_mean5": [
+    0.0021482277661561966,
+    0
+   ],
+   "co_tr_last": [
+    0.002553191501647234,
+    0
+   ],
+   "co_cos_rw": [
+    0.12024806439876556,
+    0
+   ],
+   "co_cos_sum20": [
+    0.279508501291275,
+    0
+   ]
+  }
+ },
+ "feat": {
+  "ncols": 122,
+  "only_disk": [],
+  "n_diff": 14,
+  "diff": {
+   "tr_last": [
+    0.5,
+    0
+   ],
+   "trp_last": [
+    0.011152416467666626,
+    0
+   ],
+   "tr_last3": [
+    1.0,
+    0
+   ],
+   "trp_last3": [
+    0.029003147035837173,
+    0
+   ],
+   "svd_cos_rk": [
+    1.0,
+    0
+   ],
+   "svd_ucos_rk": [
+    1.0,
+    0
+   ],
+   "svd32_cos_rk": [
+    1.0,
+    0
+   ],
+   "svd16_cos_rk": [
+    1.0,
+    0
+   ],
+   "svd24_cos_rk": [
+    1.0,
+    0
+   ],
+   "svd48_cos_rk": [
+    1.0,
+    0
+   ],
+   "svd_maxsim_rk": [
+    1.0,
+    0
+   ],
+   "svd_top3_rk": [
+    1.0,
+    0
+   ]
+  }
+ },
+ "feat_x": {
+  "ncols": 38,
+  "only_disk": [],
+  "n_diff": 24,
+  "diff": {
+   "x_clctr64": [
+    0.15912476181983948,
+    0
+   ],
+   "x_clctr64_lift": [
+    2.1029531955718994,
+    0
+   ],
+   "x_climp64": [
+    12.0,
+    0
+   ],
+   "x_clctr256": [
+    0.24428869783878326,
+    0
+   ],
+   "x_clctr256_lift": [
+    4.538407802581787,
+    0
+   ],
+   "x_climp256": [
+    17.0,
+    0
+   ],
+   "x_con05_k128": [
+    0.009315282106399536,
+    0
+   ],
+   "x_clctr64_z": [
+    2.288796730339527,
+    0
+   ],
+   "x_clctr64_lift_z": [
+    2.0558256804943085,
+    0
+   ],
+   "x_clctr256_z": [
+    3.432311713695526,
+    0
+   ],
+   "x_clctr256_lift_z": [
+    5.164675951004028,
+    0
+   ],
+   "x_con05_k128_z": [
+    0.08607450127601624,
+    0
+   ]
+  }
+ },
+ "feat_x2": {
+  "ncols": 19,
+  "only_disk": [],
+  "n_diff": 0,
+  "diff": {}
+ }
+}

parallel/lag/repro_cmp2.json

@@ -0,0 +1,188 @@
+{
+ "feat_v3": {
+  "n_cols": 169,
+  "n_cols_diff": 60,
+  "worst": [
+   [
+    "ev_co_tr_last_rk",
+    {
+     "corr": 0.98861,
+     "frac_rows_diff": 0.0011
+    }
+   ],
+   [
+    "ev_co_tr_last_z",
+    {
+     "corr": 0.98947,
+     "frac_rows_diff": 0.0011
+    }
+   ],
+   [
+    "ev_co_cos_rw_rk",
+    {
+     "corr": 0.98987,
+     "frac_rows_diff": 0.0137
+    }
+   ],
+   [
+    "ev_co_cos_max20_rk",
+    {
+     "corr": 0.99028,
+     "frac_rows_diff": 0.0135
+    }
+   ],
+   [
+    "ev_co_cos_rw_z",
+    {
+     "corr": 0.99066,
+     "frac_rows_diff": 0.0254
+    }
+   ],
+   [
+    "ev_co_cos_max20_z",
+    {
+     "corr": 0.99071,
+     "frac_rows_diff": 0.0146
+    }
+   ],
+   [
+    "ev_co_tr_max5_rk",
+    {
+     "corr": 0.99311,
+     "frac_rows_diff": 0.0043
+    }
+   ],
+   [
+    "ev_co_tr_max5_z",
+    {
+     "corr": 0.99427,
+     "frac_rows_diff": 0.0045
+    }
+   ]
+  ]
+ },
+ "feat": {
+  "n_cols": 122,
+  "n_cols_diff": 31,
+  "worst": [
+   [
+    "tr_last3_rk",
+    {
+     "corr": 0.99975,
+     "frac_rows_diff": 0.0001
+    }
+   ],
+   [
+    "trp_last3",
+    {
+     "corr": 0.99995,
+     "frac_rows_diff": 0.0001
+    }
+   ],
+   [
+    "tr_last3",
+    {
+     "corr": 0.99996,
+     "frac_rows_diff": 0.0001
+    }
+   ],
+   [
+    "tr_last3_dm",
+    {
+     "corr": 0.99996,
+     "frac_rows_diff": 0.0006
+    }
+   ],
+   [
+    "tr_last",
+    {
+     "corr": 0.99997,
+     "frac_rows_diff": 0.0
+    }
+   ],
+   [
+    "trp_last",
+    {
+     "corr": 0.99997,
+     "frac_rows_diff": 0.0
+    }
+   ],
+   [
+    "svd_cos",
+    {
+     "corr": 1.0,
+     "frac_rows_diff": 0.0
+    }
+   ],
+   [
+    "svd_ucos",
+    {
+     "corr": 1.0,
+     "frac_rows_diff": 0.0
+    }
+   ]
+  ]
+ },
+ "feat_x": {
+  "n_cols": 38,
+  "n_cols_diff": 38,
+  "worst": [
+   [
+    "x_clctr256_lift_rk",
+    {
+     "corr": 0.99607,
+     "frac_rows_diff": 0.0176
+    }
+   ],
+   [
+    "x_clctr256_lift_z",
+    {
+     "corr": 0.99638,
+     "frac_rows_diff": 0.0498
+    }
+   ],
+   [
+    "x_clctr256_lift",
+    {
+     "corr": 0.9974,
+     "frac_rows_diff": 0.0069
+    }
+   ],
+   [
+    "x_clctr256",
+    {
+     "corr": 0.99858,
+     "frac_rows_diff": 0.0069
+    }
+   ],
+   [
+    "x_clctr256_z",
+    {
+     "corr": 0.99866,
+     "frac_rows_diff": 0.0496
+    }
+   ],
+   [
+    "x_climp256",
+    {
+     "corr": 0.99869,
+     "frac_rows_diff": 0.0068
+    }
+   ],
+   [
+    "x_clctr256_rk",
+    {
+     "corr": 0.99881,
+     "frac_rows_diff": 0.0062
+    }
+   ],
+   [
+    "x_clctr64_lift_z",
+    {
+     "corr": 0.99995,
+     "frac_rows_diff": 0.0032
+    }
+   ]
+  ]
+ }
+}

parallel/lag/run2.log

@@ -0,0 +1,12 @@
+12:46:07 build C=1737763200 rows=439640 cols=170 45s
+12:46:07 wrote (439640, 170)
+12:46:13 pool 886177
+12:46:18 cooc pairs 18012356
+12:46:21 svd done
+12:46:48 build C=1737763200 rows=439640 36s
+12:46:48 wrote (439640, 125)
+val25 (439640, 39) 19s
+val25 (439640, 20) 41s
+train (1402200, 325)
+val25 {200: 0.77684, 300: 0.77724, 400: 0.77748}
+val25lag {200: 0.76434, 300: 0.76472, 400: 0.7647}

parallel/report.md

@@ -0,0 +1,135 @@
+# v2work vs v3: leakage audit and like-for-like comparison (parallel check)
+Started 23 Oct 2026 12:35 (Europe/Dublin). v2work and v3 were only read. Everything here lives in /workspace/huawei/parallel/.
+The source snapshot comes from v2work/out/solution_v3_src.zip (06:24) and is unzipped in parallel/src_snapshot/.
+Current v2work code differs from it only by the added `d{k}` fold specs (12:27 edits), so the snapshot is what produced submission_v3.
+
+## Status log
+- 12:45 Audit of the code path done (see below). Like-for-like scoring done (score_val.json). Submission check done (subcheck.json).
+
+## 1. Leakage audit of the v2work "ne" (no-eval) path
+Citations point to parallel/src_snapshot/v2work/* (identical to what produced out/submission_v3.csv).
+C is the cutoff. "Own history" is the history list that is a given input field on every row, including eval rows.
+
+**a. Are training rows only events before the cutoff? YES.**
+- v3fold.py L11–12 SPEC: f_k = first event per user in [J20+k·d, +1d).
+- In val mode, combo.py L44 trains on f1..f4 only (cut25). All target ts < Jan25.
+- Val25 is the first event per user at or after Jan25 (W=None), the same as our v3/split.
+- Final (combo.py L68, FOLDS unset in solution_v3.py) uses f1..f6, whose targets fall in Jan21..Jan26 (<Jan27).
+- build_folds.py L9–12: val25 is (Jan25, 2d), and h1..h4 are 1-day windows.
+
+**b. Are label-derived features computed only from labels before the cutoff? YES.**
+- v3feat_ne.py L109–110: P_ev and P_rows are filtered to ts<C. All CTR, impression and click stats come from these.
+- feats.py L40–41: same filter.
+- extra.py L22–24 and extra2.py L29–31 assert `CL.ts.max()<C and P_rows.ts.max()<C`.
+- Cluster CTR (extra.py L60) and user state (extra2.py L53–59) come from P_rows and P_ev.
+
+**c. Does any feature use the unlabelled eval rows? Only their own history, and only for user-level features.**
+- Item, global, co-occurrence and SVD stats come from CLg = click_log(..., T_ev.head(0)), which is train only:
+  - v3feat_ne.py L113–114 (CL vs CLg)
+  - L128 (item ip)
+  - L173 (cooc)
+  - L195 (SVD from CLg)
+  - feats.py L54–55 poolg → L65, L87, L96
+  - extra.py L31 (idf from CLg)
+  - extra2.py L35–39 (SVD)
+- User-level stats use each row's own pre-C history:
+  - v3feat_ne.py L209–212 and L222 (user fold-in onto train-only item factors)
+  - feats.py L49–50 and L117–122
+  - extra.py L36
+  - extra2.py L41
+- This is standard: every eval row carries its own history. It doesn't pool across eval rows. No eval labels exist or are used.
+- Two minor transductive touches, label-free with pre-C clicks only:
+  - extra.py L51 builds item-side vectors iC = Scg.T @ Un. Scg is train-only clicks, but Un includes the target users' folded-in vectors.
+  - extra.py L55–57 fits KMeans on Un over active users, which can include target users.
+  - Neither uses labels or post-C data. Expected effect is small.
+- write_sub.py reads evaluation.csv only for rowid order and validation.
+
+**d. Is the validation set the same as ours? YES.**
+- val25: 439,640 rows, rowid sets identical, 0 label mismatches.
+- val26: 351,380 rows, identical (score_val.json).
+
+**Does submission_v3.csv come from the ne path? YES.**
+- It equals v2work/eval_pred_v3.parquet (max abs diff 5e-9) and differs from the old eval_pred.parquet by up to 0.34.
+- eval_pred_v3 was written by `combo.py final 400 3 notime` with XF=1 XF2=1 (ne_final_v3.log: 2,105,880 rows = f1..f6, 325 feats, 3 seeds).
+- All input tables (feat_v3 at 04:06–04:24, feat at 04:06–04:12, feat_x at 04:06–04:15, feat_x2 at 04:16–04:27) were written by the ne_* runs after the no-eval edits (feats.py and v3feat_ne.py at 04:03).
+
+**FLAG: our own v3 is less strict than v2work here.**
+- v3/features.py builds item stats (L127), co-occurrence (L172) and the SVD matrix (L194) from CL, which includes every target row's own history.
+- For eval predictions this means our submission_v2.csv pools all eval rows' (pre-cutoff, label-free) histories into item-level stats.
+- That's the same pattern Grok Bot labelled a leak and superseded in its old v2.
+- It's label-free and pre-Jan27, so it's not label leakage. But it is transductive use of the eval rows. Whether that's allowed is Eric's call.
+
+## 2. Like-for-like scoring (our metric: global ROC AUC, v4/common.report), identical rows
+v3 preds come from re-predicting the saved models v3/out/full25.txt and full26_lr05.txt in parquet row order.
+The saved v3 *_valpred.npy files have an unrecoverable row order because of an unstable sort in v3/train.py. The re-prediction reproduces 0.75554 and 0.77444.
+
+| set | model | AUC | eval-weighted | stale ≤1d | stale 1–2d | event AUC* | Spearman vs v3 |
+|---|---|---|---|---|---|---|---|
+| val25 (439,640) | **v3 (ours)** | 0.75554 | 0.75554 | 0.76059 | 0.73424 | 0.64894 | – |
+| val25 | **v2work ne notime 325f** (combo_val25_notime.npy, e-fold run, 300 rounds) | **0.77749** | 0.77748 | 0.78199 | 0.75830 | 0.65403 | 0.899 |
+| val25 | v2work notime_x2 (pre-ne config, 04:26) | 0.77536 | 0.77535 | 0.77968 | 0.75708 | 0.65328 | 0.899 |
+| val25 | v2work screen, base, 25%-user train, 1 seed | 0.77174 | 0.77173 | 0.77642 | 0.75176 | 0.59153* | 0.888 |
+| val26 (351,380) | **v3 (ours)** | 0.77444 | 0.77444 | 0.77444 | – | 0.65666 | – |
+| val26 | v2work screen, base, 25%-user train | **0.78921** | 0.78921 | 0.78921 | – | 0.59846* | 0.912 |
+
+*Event AUC = AUC of max(y) vs sum(p) per event. The screen preds are raw scores, so their sum isn't comparable; ignore those two event numbers.
+
+Notes:
+- The final config (f-folds, 400 rounds) scored 0.77748 in ne_combo25.log, but its preds were overwritten by the e-fold run (0.77749 at 300). The two are equivalent proxies.
+- There are no full-data v2work val26 preds. The 25% screen alone beats our full v3 by +0.015.
+- The gain is uniform across staleness buckets: +0.021 at ≤1d and +0.024 at 1–2d. It isn't concentrated where leakage would show.
+
+**Final submissions:** Spearman(v2work/out/submission_v3.csv, /workspace/huawei/submission_v2.csv) = **0.918** (Pearson 0.929) over 529,390 rows (all joined). Mean p is 0.0625 vs 0.0616.
+
+## 3. Reproduction and lag-1d leak test (parallel/lag/, one job at a time, 3 threads, seed 42, deterministic)
+**Reproduction.**
+- Retrained v2work's exact final config (snapshot combo.py PARAMS, notime, XF=1 XF2=1, f1..f4, 400 rounds) on v2work's on-disk tables, read only.
+- val25 AUC was 0.77684, 0.77724 and **0.77748** at 200, 300 and 400 rounds. That's identical to ne_combo25.log, so 0.7775 is real and reproducible.
+- I also rebuilt the val25 tables from the snapshot code (lag/repro_cmp2.json):
+  - feat_x2 is exact.
+  - feat (v2) has 31 of 122 cols with tiny diffs (corr ≥0.9997).
+  - feat_v3 has 60 of 169 co-occurrence/rank cols differing on ≤2.5% of rows (corr ≥0.989).
+  - feat_x has clctr cols differing on ≤5% of rows (corr ≥0.996).
+  - These diffs are consistent with thread-dependent randomized SVD/KMeans and tie-breaking. They don't point to a different (eval-pooled) build.
+
+**Lag-1d test.**
+- Same target rows (val25 = first event per user at or after Jan25, labels unchanged). All four feature builders were rerun with the cutoff frozen at **Jan24**, so no stat or label from [Jan24, Jan25) is used.
+- Predicted with the same model (lag/lagtest.py):
+
+| val25 features | AUC | stale ≤1d | stale 1–2d | event AUC |
+|---|---|---|---|---|
+| as built (C=Jan25) | 0.77748 | 0.78184 | 0.75895 | 0.65358 |
+| lag-1d (C=Jan24) | 0.76470 | 0.76946 | 0.74430 | 0.64837 |
+
+- The drop is −0.0128, gradual, with no cliff. It's smaller than v3's own lag-1d drop (0.7556→0.7391, −0.0165; v3/out/leakcheck.jsonl).
+- Even with 1-day-stale features, v2work (0.7647) beats our v3 with fresh features (0.7555).
+- A target-day leak would collapse towards ~0.65–0.70 when the target day is removed. That doesn't happen.
+
+## 4. Verdict
+**v2work's ne pipeline is genuinely better than our v3.**
+- val25: +0.022 AUC (0.7775 vs 0.7555) on identical rows and labels.
+- val26 screen: +0.015, even with a 25% train sample.
+- The gain holds in every staleness bucket and survives the lag-1d test.
+
+The audit found no label leakage:
+- Training targets are before C.
+- All label stats come from ts<C, with asserts.
+- Item and global stats are train only.
+- Eval rows contribute only their own given pre-cutoff history to user-level features.
+
+Grok Bot's claims all check out:
+- The ne variant files.
+- 0.7775 from ne_combo25 and ne_combo25_eft.
+- submission_v3.csv built by the 06:30 combo final run (it equals eval_pred_v3.parquet).
+- The old v2work submission_v2 is superseded.
+
+Correlation with our submission: Spearman 0.918.
+
+**Flags for Eric:**
+1. **v2work/out/submission_v3.csv appears to have been uploaded already.** v2work/out/submission_log.csv has a row "submission_v3.csv.zip (public score) … 0.7775, 0.788757", so public AUC was 0.7888. This parallel check didn't upload or submit anything.
+2. Our v3 (and so /workspace/huawei/submission_v2.csv) builds item, co-occurrence and SVD stats from all target and eval rows' histories (v3/features.py L127/L172/L194). That's label-free but transductive, and stricter rules would call it eval-row use. v2work removed this.
+3. Minor: v2work's extra.py L51 and L55–57 still fold target users' (pre-C, label-free) vectors into item-side aggregates and KMeans.
+
+**Final CSV:** it exists. v2work/out/submission_v3.csv: sha256 466f3d64…, 529,390 rows, written 06:37, plus solution_v3.py and src zip.
+- Rebuilding from scratch at ≤3 threads would take about 40–50 min: feature tables ~25 min, then final fit on 2.1M rows × 3 seeds × 400 rounds ~12–20 min (the 1.4M-row fit here took ~3 min).
+- If all tables are reused, only the final fit is needed: ~15–20 min. I didn't build one.

parallel/score.log

@@ -0,0 +1,102 @@
+{
+ "val25": {
+  "rows_v3": 439640,
+  "rows_v2work": 439640,
+  "rows_common": 439640,
+  "only_v3": 0,
+  "only_v2work": 0,
+  "label_mismatch": 0,
+  "v3 (full25.txt)": {
+   "auc": 0.75554,
+   "n": 439640,
+   "auc_<=1d": 0.76059,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.73424,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.75554,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.64894
+  },
+  "v2work_notime_eft_ne (combo_val25_notime.npy, 300r)": {
+   "auc": 0.77749,
+   "n": 439640,
+   "auc_<=1d": 0.78199,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.7583,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77748,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.65403,
+   "spearman_vs_v3": 0.899
+  },
+  "v2work_notime_x2 (04:26)": {
+   "auc": 0.77536,
+   "n": 439640,
+   "auc_<=1d": 0.77968,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.75708,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77535,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.65328,
+   "spearman_vs_v3": 0.8989
+  },
+  "v2work_screen_base_25pct": {
+   "auc": 0.77174,
+   "n": 439640,
+   "auc_<=1d": 0.77642,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.75176,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77173,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.59153,
+   "spearman_vs_v3": 0.8875
+  }
+ }
+}
+{
+ "val26": {
+  "rows_v3": 351380,
+  "rows_v2work": 351380,
+  "rows_common": 351380,
+  "only_v3": 0,
+  "only_v2work": 0,
+  "label_mismatch": 0,
+  "v3 (full26_lr05.txt)": {
+   "auc": 0.77444,
+   "n": 351380,
+   "auc_<=1d": 0.77444,
+   "share_<=1d": 1.0,
+   "auc_1-2d": null,
+   "share_1-2d": 0.0,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77444,
+   "evalw_buckets_present": 1,
+   "auc_event": 0.65666
+  },
+  "v2work_screen_base_25pct (scr_base_c26.npy)": {
+   "auc": 0.78921,
+   "n": 351380,
+   "auc_<=1d": 0.78921,
+   "share_<=1d": 1.0,
+   "auc_1-2d": null,
+   "share_1-2d": 0.0,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.78921,
+   "evalw_buckets_present": 1,
+   "auc_event": 0.59846,
+   "spearman_vs_v3": 0.9115
+  }
+ }
+}

parallel/score.py

@@ -0,0 +1,29 @@
+"""Like-for-like scoring of saved val predictions (v3 vs v2work) on identical rows. Read-only on v2work/v3."""
+import sys, json, numpy as np, polars as pl
+sys.path.insert(0, '/workspace/huawei/v4')
+from common import report
+from scipy.stats import spearmanr
+V2='/workspace/huawei/v2work'; V3='/workspace/huawei/v3'
+out={}
+for cut, v3pred, v2preds in [(25, 'full25.txt', {'v2work_notime_eft_ne (combo_val25_notime.npy, 300r)':'combo_val25_notime.npy',
+                                                        'v2work_notime_x2 (04:26)':'combo_val25_notime_x2.npy',
+                                                        'v2work_screen_base_25pct':'scr_base_c25.npy'}),
+                             (26, 'full26_lr05.txt', {'v2work_screen_base_25pct (scr_base_c26.npy)':'scr_base_c26.npy'})]:
+    # v3 val table: load_table sorts by (fold, eid) -> rows sorted by eid with stable intra-event order
+    v3 = pl.read_parquet(f'{V3}/feat/val{cut}.parquet', columns=['rowid','eid','clicked','tm_stale'])
+    v3 = v3.join(pl.read_parquet(f'/workspace/huawei/parallel/v3_val{cut}_pred.parquet'), on='rowid', how='left')  # re-predicted (v3_repredict.py)
+    assert v3['p_v3'].null_count()==0
+    v2 = pl.read_parquet(f'{V2}/feat_v3/val{cut}.parquet', columns=['rowid','eid','clicked','tm_stale'])
+    for nm,f in v2preds.items():
+        p=np.load(f'{V2}/{f}'); assert len(p)==v2.height, (f,len(p),v2.height); v2=v2.with_columns(pl.Series(nm,p))
+    rs3=set(v3['rowid'].to_list()); rs2=set(v2['rowid'].to_list())
+    j = v3.join(v2.drop('eid','tm_stale').rename({'clicked':'clicked2'}), on='rowid', how='inner')
+    res={'rows_v3':v3.height,'rows_v2work':v2.height,'rows_common':j.height,'only_v3':len(rs3-rs2),'only_v2work':len(rs2-rs3),
+         'label_mismatch':int((j['clicked']!=j['clicked2']).sum())}
+    res['v3 ('+v3pred+')']=report(j, j['p_v3'].to_numpy())
+    for nm in v2preds:
+        res[nm]=report(j, j[nm].to_numpy())
+        res[nm]['spearman_vs_v3']=round(float(spearmanr(j[nm].to_numpy(), j['p_v3'].to_numpy()).statistic),4)
+    out[f'val{cut}']=res
+    print(json.dumps({f'val{cut}':res},indent=1),flush=True)
+json.dump(out,open('/workspace/huawei/parallel/score_val.json','w'),indent=1)

parallel/score_val.json

@@ -0,0 +1,100 @@
+{
+ "val25": {
+  "rows_v3": 439640,
+  "rows_v2work": 439640,
+  "rows_common": 439640,
+  "only_v3": 0,
+  "only_v2work": 0,
+  "label_mismatch": 0,
+  "v3 (full25.txt)": {
+   "auc": 0.75554,
+   "n": 439640,
+   "auc_<=1d": 0.76059,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.73424,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.75554,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.64894
+  },
+  "v2work_notime_eft_ne (combo_val25_notime.npy, 300r)": {
+   "auc": 0.77749,
+   "n": 439640,
+   "auc_<=1d": 0.78199,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.7583,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77748,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.65403,
+   "spearman_vs_v3": 0.899
+  },
+  "v2work_notime_x2 (04:26)": {
+   "auc": 0.77536,
+   "n": 439640,
+   "auc_<=1d": 0.77968,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.75708,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77535,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.65328,
+   "spearman_vs_v3": 0.8989
+  },
+  "v2work_screen_base_25pct": {
+   "auc": 0.77174,
+   "n": 439640,
+   "auc_<=1d": 0.77642,
+   "share_<=1d": 0.80134,
+   "auc_1-2d": 0.75176,
+   "share_1-2d": 0.19866,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77173,
+   "evalw_buckets_present": 2,
+   "auc_event": 0.59153,
+   "spearman_vs_v3": 0.8875
+  }
+ },
+ "val26": {
+  "rows_v3": 351380,
+  "rows_v2work": 351380,
+  "rows_common": 351380,
+  "only_v3": 0,
+  "only_v2work": 0,
+  "label_mismatch": 0,
+  "v3 (full26_lr05.txt)": {
+   "auc": 0.77444,
+   "n": 351380,
+   "auc_<=1d": 0.77444,
+   "share_<=1d": 1.0,
+   "auc_1-2d": null,
+   "share_1-2d": 0.0,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.77444,
+   "evalw_buckets_present": 1,
+   "auc_event": 0.65666
+  },
+  "v2work_screen_base_25pct (scr_base_c26.npy)": {
+   "auc": 0.78921,
+   "n": 351380,
+   "auc_<=1d": 0.78921,
+   "share_<=1d": 1.0,
+   "auc_1-2d": null,
+   "share_1-2d": 0.0,
+   "auc_>2d": null,
+   "share_>2d": 0.0,
+   "auc_evalw": 0.78921,
+   "evalw_buckets_present": 1,
+   "auc_event": 0.59846,
+   "spearman_vs_v3": 0.9115
+  }
+ }
+}

parallel/src_snapshot/v2work/blend_val.py

@@ -0,0 +1,44 @@
+"""Compare v2 (my features) vs v3 features vs blend on the shared validation:
+cutoff Jan25, targets = first event per user in [Jan25, Jan27); training rows = first-event folds with cutoffs Jan21..Jan24 (W=1d)."""
+import sys, os, numpy as np, polars as pl, lightgbm as lgb, json
+sys.path.insert(0,'/workspace/huawei/v2work'); sys.path.insert(0,'/workspace/huawei/v3')
+from feats import DROP
+from features import feature_cols, GROUPS
+from sklearn.metrics import roc_auc_score
+from scipy.stats import rankdata
+P2='/workspace/huawei/v2work/feat'; P3='/workspace/huawei/v3/feat'
+def fit_pred(tr, va, feats, params, rounds, ev_every=25):
+    Xtr=tr.select(feats).to_numpy().astype(np.float32); Xva=va.select(feats).to_numpy().astype(np.float32); y=va['clicked'].to_numpy()
+    m=lgb.train(params,lgb.Dataset(Xtr,tr['clicked'].to_numpy(),feature_name=feats),num_boost_round=rounds)
+    curve=[(k,roc_auc_score(y,m.predict(Xva,num_iteration=k))) for k in range(ev_every,rounds+1,ev_every)]
+    return m,curve,Xva
+base=dict(objective='binary',num_leaves=63,min_data_in_leaf=200,feature_fraction=0.7,bagging_fraction=0.8,bagging_freq=1,lambda_l2=10.0,verbose=-1,seed=42,num_threads=8)
+# ---- v3 features
+tr3=pl.concat([pl.read_parquet(f'{P3}/f{k}.parquet') for k in (1,2,3,4)],how='diagonal_relaxed')
+va3=pl.read_parquet(f'{P3}/val25.parquet')
+f3=feature_cols(tr3,GROUPS)
+m3,c3,X3=fit_pred(tr3,va3,f3,dict(base,learning_rate=0.05),500)
+print('v3 curve',[(k,round(a,5)) for k,a in c3],flush=True)
+b3=max(c3,key=lambda x:x[1]); p3=m3.predict(X3,num_iteration=b3[0])
+# ---- v2 features
+tr2=pl.concat([pl.read_parquet(f'{P2}/h{k}.parquet') for k in (1,2,3,4)],how='diagonal_relaxed')
+va2=pl.read_parquet(f'{P2}/val25.parquet')
+f2=[c for c in tr2.columns if c not in DROP]
+m2,c2,X2=fit_pred(tr2,va2,f2,dict(base,learning_rate=0.05),500)
+print('v2 curve',[(k,round(a,5)) for k,a in c2],flush=True)
+b2=max(c2,key=lambda x:x[1]); p2=m2.predict(X2,num_iteration=b2[0])
+# ---- combined feature set (v3 + v2 features not in v3), joined on rowid
+extra=va2.select(['rowid']+[c for c in f2]).rename({c:'v2_'+c for c in f2})
+trc=tr3.join(tr2.select(['rowid']+f2).rename({c:'v2_'+c for c in f2}),on='rowid',how='left')
+vac=va3.join(extra,on='rowid',how='left')
+fc=f3+['v2_'+c for c in f2]
+mc,cc,Xc=fit_pred(trc,vac,fc,dict(base,learning_rate=0.05),500)
+print('combined curve',[(k,round(a,5)) for k,a in cc],flush=True)
+# ---- blends on rowid
+a=pl.DataFrame({'rowid':va3['rowid'],'p3':p3,'y':va3['clicked']}).join(pl.DataFrame({'rowid':va2['rowid'],'p2':p2}),on='rowid')
+y=a['y'].to_numpy(); r3=rankdata(a['p3'])/len(a); r2=rankdata(a['p2'])/len(a)
+res=dict(v3=b3,v2=b2,combined=max(cc,key=lambda x:x[1]))
+for w in (0.5,0.6,0.7,0.8,0.9): res[f'rankblend_w3={w}']=roc_auc_score(y,w*r3+(1-w)*r2)
+print(json.dumps(res,indent=1))
+imp=sorted(zip(fc,mc.feature_importance('gain')),key=lambda x:-x[1]); tot=sum(v for _,v in imp)
+print('combined top', [(f,round(v/tot,4)) for f,v in imp[:25]])

parallel/src_snapshot/v2work/build_folds.py

@@ -0,0 +1,27 @@
+import polars as pl, sys, os
+sys.path.insert(0,'/workspace/huawei/v2work')
+from feats import load, first_events, build, log
+D='/workspace/huawei/v2work'; os.makedirs(f'{D}/feat',exist_ok=True)
+TE,TR,EE,ER,NI=load(D)
+DAY=86400; J20=1737331200
+END=J20+7*DAY; VC=J20+4*DAY
+# validation: V = first event/user in [Jan24,Jan27); training folds end before Jan24
+specs=[(VC,END-VC,'V')]+[(J20+k*DAY,VC-(J20+k*DAY),f'g{k}') for k in (1,2,3)]
+if len(sys.argv)>1 and sys.argv[1]=='c25':
+    specs=[(J20+5*DAY,2*DAY,'val25')]+[(J20+k*DAY,DAY,f'h{k}') for k in (1,2,3,4)]
+if len(sys.argv)>1 and sys.argv[1]=='c26':
+    specs=[(J20+6*DAY,DAY,'val26')]+[(J20+k*DAY,DAY,f'h{k}') for k in (5,)]+[('eval',None,'eval')]
+if len(sys.argv)>1 and sys.argv[1]=='eft':
+    specs=[(J20+k*DAY,(J20+5*DAY)-(J20+k*DAY),f'e{k}') for k in (1,2,3,4)]+[(J20+k*DAY,END-(J20+k*DAY),f'E{k}') for k in (1,2,3,4,5,6)]
+if len(sys.argv)>1 and sys.argv[1]=='final':
+    specs=[(J20+k*DAY,END-(J20+k*DAY),f'F{k}') for k in (1,2,3,4,5,6)]+[('eval',None,'eval')]
+for C,W,name in specs:
+    out=f'{D}/feat/{name}.parquet'
+    if os.path.exists(out): continue
+    if C=='eval':
+        C=J20+7*DAY; T_ev=EE; T_rows=ER
+    else:
+        T_ev=first_events(TE,C,W); T_rows=TR.join(T_ev.select('eid'),on='eid')
+    R=build(C,T_ev,T_rows,TE,TR,NI)
+    R.drop('h','h_ts').write_parquet(out)
+    log('wrote',name,R.shape)

parallel/src_snapshot/v2work/combo.py

@@ -0,0 +1,85 @@
+"""Combined (v3 features + v2 features) LightGBM. Modes:
+  python combo.py val 25|26 [variant]   -> fixed-round curve on val25/val26, segment + weekday/weekend AUCs
+  python combo.py final N_ITER SEEDS    -> fit on all 6 cutoff folds (Jan21..Jan26), predict eval (cutoff Jan27)
+variants: base | notime (drop all time/gap feats) | wkw (weekend train events weight 0.3) | notime_wkw"""
+import sys, os, json, numpy as np, polars as pl, lightgbm as lgb
+sys.path.insert(0,'/workspace/huawei/v2work'); sys.path.insert(0,'/workspace/huawei/v3')
+from feats import DROP
+from features import feature_cols, GROUPS
+from sklearn.metrics import roc_auc_score
+P2='/workspace/huawei/v2work/feat'; P3='/workspace/huawei/v2work/feat_v3'
+V2_OF={'f1':'h1','f2':'h2','f3':'h3','f4':'h4','f5':'h5','f6':'val26','val25':'val25','val26':'val26','eval':'eval'}
+V2_OF.update({f'{p}{k}':f'{p}{k}' for p in 'eE' for k in range(1,7)})
+EFT=os.environ.get('FOLDS')=='eft'   # eval-like multi-day target windows (e1..e4 -> Jan25, E1..E6 -> Jan27)
+OUTP=os.environ.get('OUTP','eval_pred')
+TIME=['tm_stale','tm_hour','tm_tb','tm_dt','v2_tb','v2_dt','v2_stale','v2_hour']
+def load(names):
+    out=[]
+    for n in names:
+        a=pl.read_parquet(f'{P3}/{n}.parquet'); b=pl.read_parquet(f'{P2}/{V2_OF[n]}.parquet')
+        f2=[c for c in b.columns if c not in DROP]
+        b=b.select(['rowid']+f2).rename({c:'v2_'+c for c in f2})
+        a=a.join(b,on='rowid',how='left')
+        if os.environ.get('XF'):
+            a=a.join(pl.read_parquet(f'/workspace/huawei/v2work/feat_x/{n}.parquet'),on='rowid',how='left')
+        if os.environ.get('XF2'):
+            a=a.join(pl.read_parquet(f'/workspace/huawei/v2work/feat_x2/{n}.parquet'),on='rowid',how='left')
+        out.append(a)
+    return pl.concat(out,how='diagonal_relaxed')
+def featlist(df,variant):
+    f=feature_cols(df,GROUPS)+[c for c in df.columns if c.startswith(('v2_','x_','r_','s_'))]
+    if 'notime' in variant: f=[c for c in f if c not in TIME]
+    return f
+def weights(df,variant):
+    if 'wkw' not in variant: return None
+    dow=((df['ts'].to_numpy()//86400)+3)%7   # 0=Mon (1970-01-01 was Thu)
+    return np.where(dow>=5,0.3,1.0)
+PARAMS=dict(objective='binary',learning_rate=0.05,num_leaves=63,min_data_in_leaf=200,feature_fraction=0.7,bagging_fraction=0.8,
+            bagging_freq=1,lambda_l2=10.0,verbose=-1,seed=42,num_threads=8)
+mode=sys.argv[1]
+if mode=='val':
+    cut=int(sys.argv[2]); variant=sys.argv[3] if len(sys.argv)>3 else 'base'
+    import gc
+    feats=None; Xs=[]; ys=[]
+    for k in range(1,cut-20):
+        t=load([f'e{k}' if EFT else f'f{k}'])
+        if feats is None: feats=featlist(t,variant)
+        Xs.append(t.select(feats).to_numpy().astype(np.float32)); ys.append(t['clicked'].to_numpy()); del t; gc.collect()
+    X=np.concatenate(Xs); del Xs; gc.collect()
+    ds=lgb.Dataset(X,np.concatenate(ys),feature_name=feats,free_raw_data=True).construct(); del X; gc.collect()
+    m=lgb.train(PARAMS,ds,num_boost_round=450); del ds; gc.collect()
+    va=load([f'val{cut}'])
+    Xv=va.select(feats).to_numpy().astype(np.float32); y=va['clicked'].to_numpy()
+    curve={k:roc_auc_score(y,m.predict(Xv,num_iteration=k)) for k in range(50,451,50)}
+    best=max(curve,key=curve.get); p=m.predict(Xv,num_iteration=300)
+    dow=((va['ts'].to_numpy()//86400)+3)%7
+    cold=va['us_cold'].to_numpy()==1; noh=va['us_nohist'].to_numpy()==1
+    segs={'all':np.ones(len(y),bool),'warm':~cold,'history_only':cold&~noh,'empty_history':noh,'weekday':dow<5,'weekend':dow>=5,'stale_lt1d':va['tm_stale'].to_numpy()<86400,'stale_ge1d':va['tm_stale'].to_numpy()>=86400}
+    seg={k:(int(s.sum()),round(roc_auc_score(y[s],p[s]),5) if s.sum()>0 and y[s].min()<y[s].max() else None) for k,s in segs.items()}
+    g=m.feature_importance('gain'); top=[(feats[i],round(float(g[i]/g.sum()),4)) for i in np.argsort(-g)[:20]]
+    res=dict(cut=cut,variant=variant+('_eft' if EFT else '')+('_ne'),n_feats=len(feats),curve={k:round(v,5) for k,v in curve.items()},best=best,seg_at300=seg,top=top)
+    print(json.dumps(res),flush=True)
+    open('/workspace/huawei/v2work/combo_results.jsonl','a').write(json.dumps(res)+'\n')
+    np.save(f'/workspace/huawei/v2work/combo_val{cut}_{variant}.npy',p)
+elif mode=='final':
+    import gc
+    n_iter=int(sys.argv[2]); seeds=int(sys.argv[3]); variant=sys.argv[4] if len(sys.argv)>4 else 'base'
+    feats=None; Xs=[]; ys=[]; ws=[]
+    for k in range(1,7):   # load fold by fold -> float32 numpy to keep memory low
+        t=load([f'E{k}' if EFT else f'f{k}'])
+        if feats is None: feats=featlist(t,variant)
+        Xs.append(t.select(feats).to_numpy().astype(np.float32)); ys.append(t['clicked'].to_numpy())
+        w=weights(t,variant); ws.append(w if w is not None else np.ones(len(t)))
+        del t; gc.collect()
+    X=np.concatenate(Xs); del Xs; y=np.concatenate(ys); w=np.concatenate(ws) if 'wkw' in variant else None
+    print('train rows',len(y),'feats',len(feats),flush=True)
+    ds=lgb.Dataset(X,y,weight=w,feature_name=feats,free_raw_data=True).construct(); del X; gc.collect()
+    preds=[]; ms=[]
+    for s in range(seeds):
+        m=lgb.train(dict(PARAMS,seed=42+s),ds,num_boost_round=n_iter)
+        m.save_model(f'/workspace/huawei/v2work/{OUTP}_model_s{s}.txt'); ms.append(m); print('seed',s,'done',flush=True)
+    del ds; gc.collect()
+    ev=load(['eval']); Xe=ev.select(feats).to_numpy().astype(np.float32)
+    p=np.mean([m.predict(Xe) for m in ms],0)   # plain probability average across seeds; no per-event normalisation
+    pl.DataFrame({'rowid':ev['rowid'],'p':p}).write_parquet(f'/workspace/huawei/v2work/{OUTP}.parquet')
+    g=m.feature_importance('gain'); print('top',[(feats[i],round(float(g[i]/g.sum()),4)) for i in np.argsort(-g)[:20]])

parallel/src_snapshot/v2work/emb_exp.py

@@ -0,0 +1,38 @@
+import polars as pl, numpy as np, sys
+sys.path.insert(0,'/workspace/huawei/v2work')
+from feats import load, first_events
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+from sklearn.metrics import roc_auc_score
+TE,TR,EE,ER,NI=load('/workspace/huawei/v2work')
+DAY=86400; C=1737331200+4*DAY
+T_ev=first_events(TE,C,DAY); T_rows=TR.join(T_ev.select('eid'),on='eid')
+P_ev=TE.filter(pl.col('ts')<C); P=TR.join(P_ev.select('eid','user','ts'),on='eid')
+hc=lambda E: E.select('user','h','h_ts').explode(['h','h_ts']).drop_nulls().rename({'h':'ii','h_ts':'ts'})
+pool=pl.concat([hc(P_ev),hc(T_ev),P.filter(pl.col('clicked')==1).select('user','ii','ts')]).unique()
+us=pool.group_by('user','ii').agg(pl.len().alias('n'))
+uidx=us.select('user').unique().with_row_index('ur'); m=us.join(uidx,on='user')
+R=T_rows.join(T_ev.select('eid','user','h'),on='eid').sort('eid')
+y=R['clicked'].to_numpy(); eid=R['eid'].to_numpy()
+ex=R.with_row_index('r').select('r','h').with_columns(pl.int_ranges(pl.col('h').list.len(),0,-1).alias('pos')).explode(['h','pos']).drop_nulls()
+rr=ex['r'].to_numpy(); hh=ex['h'].to_numpy(); pos=ex['pos'].to_numpy()
+um=R.select('user').join(uidx,on='user',how='left')['ur'].fill_null(0).to_numpy()
+def dm(x):
+    s=pl.DataFrame({'e':eid,'x':x}).with_columns((pl.col('x')-pl.col('x').mean().over('e')).alias('d'))['d'].to_numpy(); return s
+def ev(name,V,U=None,decay=0.85):
+    hv=np.zeros((len(R),V.shape[1]),np.float32); np.add.at(hv,rr,V[hh]*(decay**(pos-1))[:,None])
+    hvn=hv/(np.linalg.norm(hv,axis=1,keepdims=True)+1e-6); c=V[R['ii'].to_numpy()]
+    s=(hvn*c).sum(1); out=f'{name}: hist cos {roc_auc_score(y,s):.4f} dm {roc_auc_score(y,dm(s)):.4f}'
+    if U is not None:
+        s2=(U[um]*c).sum(1); out+=f' | ucos {roc_auc_score(y,s2):.4f} dm {roc_auc_score(y,dm(s2)):.4f}'
+    print(out,flush=True)
+ii=m['ii'].to_numpy(); ur=m['ur'].to_numpy()
+df_=np.bincount(ii,minlength=NI); idf=np.log((len(uidx)+1)/(df_+1)).astype(np.float32)
+for wname,w in [('bin',np.ones(len(ii),np.float32)),('idf',idf[ii])]:
+    X=sparse.csr_matrix((w,(ur,ii)),shape=(len(uidx),NI)); rn=np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel())+1e-6; X=sparse.diags(1/rn)@X
+    for k in (128,256):
+        U,S,Vt=randomized_svd(X,k,random_state=0,n_iter=4)
+        for a in (0.0,):
+            V=(Vt.T*S**a).astype(np.float32); Vn=V/(np.linalg.norm(V,axis=1,keepdims=True)+1e-6)
+            Uf=(U*S**a).astype(np.float32); Uf/=np.linalg.norm(Uf,axis=1,keepdims=True)+1e-6
+            ev(f'{wname} k={k} a={a}',Vn,Uf)

parallel/src_snapshot/v2work/explore.py

@@ -0,0 +1,26 @@
+import polars as pl
+tr=pl.read_parquet('train_rows.parquet'); te=pl.read_parquet('train_events.parquet')
+er=pl.read_parquet('evaluation_rows.parquet'); ee=pl.read_parquet('evaluation_events.parquet')
+print(tr.shape, te.shape, er.shape, ee.shape)
+print('uniq eid', tr['eid'].n_unique(), te['eid'].n_unique(), ee['eid'].n_unique())
+print('ctr', tr['clicked'].mean())
+ev=tr.group_by('eid').agg(pl.col('clicked').sum().alias('c'))
+print('clicks/event dist', ev['c'].value_counts().sort('c'))
+print('users train', te['user'].n_unique(), 'eval', ee['user'].n_unique(), 'eval users in train', ee['user'].is_in(te['user'].implode()).sum())
+print('items train', tr['item'].n_unique(), 'eval', er['item'].n_unique(), 'eval items in train', er['item'].unique().is_in(tr['item'].implode()).sum())
+print('eval rows with item seen', er['item'].is_in(tr['item'].implode()).mean())
+print('hist len train', te['h_items'].list.len().describe())
+print('hist len eval', ee['h_items'].list.len().describe())
+print('events/user', te.group_by('user').len()['len'].describe())
+print('ts range', te['ts'].min(), te['ts'].max(), ee['ts'].min(), ee['ts'].max())
+print(ee.with_columns(pl.from_epoch('ts')).select(pl.col('ts').dt.date()).to_series().value_counts().sort('ts'))
+print(te.with_columns(pl.from_epoch('ts')).select(pl.col('ts').dt.date()).to_series().value_counts().sort('ts'))
+# candidate in history rate
+x=tr.join(te.select('eid','h_items'),on='eid')
+x=x.with_columns(pl.col('h_items').list.contains(pl.col('item')).alias('inh'))
+print(x.group_by('inh').agg(pl.len(), pl.col('clicked').mean()))
+# items per event duplicate?
+print('dup items in event', tr.group_by('eid').agg(pl.col('item').n_unique().alias('n'))['n'].value_counts())
+# item impressions distribution
+ic=tr.group_by('item').agg(pl.len().alias('n'), pl.col('clicked').mean().alias('ctr'))
+print(ic.describe())

parallel/src_snapshot/v2work/extra.py

@@ -0,0 +1,78 @@
+"""Extra leak-free features on v3 data (all stats from data with ts < C).
+Usage: python extra.py NAME...   (names: f1..f6, val25, val26, eval)  -> v2work/feat_x/NAME.parquet
+ - con variants: k=32/128, recency-weighted (tau=2d) click/non-click centroids, item-side con (users who clicked i vs skipped i)
+ - user-cluster item CTR (kmeans on user SVD vectors, 64 & 256 clusters), smoothed toward item CTR
+ - within-event z-score and rank for each"""
+import sys, os, time, numpy as np, polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+from sklearn.cluster import MiniBatchKMeans
+sys.path.insert(0, '/workspace/huawei/v3')
+from features import load, click_log
+from split import J20, DAY, first_events
+OUT = '/workspace/huawei/v2work/feat_x'; os.makedirs(OUT, exist_ok=True)
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'),
+        **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}}
+def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
+def csr(u, i, w): return sparse.csr_matrix((w.astype(np.float32), (u, i)), shape=(NU, NI))
+def build(C, T_ev, T_rows):
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    CL = click_log(C, P_ev, P_rows, T_ev)            # user-level: train data + each target row's own history
+    CLg = click_log(C, P_ev, P_rows, T_ev.head(0))   # global/item-level: train data only
+    assert CL['ts'].max() < C and P_rows['ts'].max() < C
+    R = T_rows.join(T_ev.select('eid', 'u'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii')
+    uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+    cg = CLg.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = csr(cg['u'].to_numpy(), cg['ii'].to_numpy(), np.ones(cg.height)); Scg = Xc.copy()
+    cnt = CL.group_by('u', 'ii').agg(pl.len().alias('n'), pl.col('ts').max().alias('lt'))
+    cu = cnt['u'].to_numpy(); ci = cnt['ii'].to_numpy()
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc
+    Xu = csr(cu, ci, np.ones(cnt.height)) @ sparse.diags(idf); Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu
+    neg = P_rows.filter(pl.col('clicked') == 0)
+    nu_ = neg['u'].to_numpy(); ni_ = neg['ii'].to_numpy()
+    Sc = csr(cu, ci, np.ones(cnt.height)); Sn = csr(nu_, ni_, np.ones(neg.height))
+    rc = np.exp(-(C - cnt['lt'].to_numpy()) / (2 * DAY)); rn = np.exp(-(C - neg['ts'].to_numpy()) / (2 * DAY))
+    Scr = csr(cu, ci, rc); Snr = csr(nu_, ni_, rn)
+    F = {}
+    for k in (32, 64, 128):
+        U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)
+        Vn = nrm(Vt.T * np.sqrt(S)); Un = nrm(np.asarray(Xu @ (Vt.T / np.sqrt(S)))); cand = Vn[ii]  # user fold-in
+        nC = nrm(np.asarray(Sc @ Vn)); nN = nrm(np.asarray(Sn @ Vn))
+        F[f'x_con05_k{k}'] = (nrm(nC - 0.5 * nN)[uu] * cand).sum(1)
+        if k == 64:
+            nCr = nrm(np.asarray(Scr @ Vn)); nNr = nrm(np.asarray(Snr @ Vn))
+            for a, nm in ((0.5, '05'), (1.0, '1')):
+                F[f'x_conr{nm}_k64'] = (nrm(nCr - a * nNr)[uu] * cand).sum(1)
+            F['x_crcos_k64'] = (nCr[uu] * cand).sum(1)
+            # item side: centroid of users who clicked item minus users who skipped it, vs target user vector
+            iC = nrm(np.asarray(Scg.T @ Un)); iN = nrm(np.asarray(Sn.T @ Un))  # item side: train-only clicks
+            F['x_icon05_k64'] = (nrm(iC - 0.5 * iN)[ii] * Un[uu]).sum(1)
+            F['x_icos_k64'] = (iC[ii] * Un[uu]).sum(1)
+            # user-cluster CTR
+            act = np.unique(np.concatenate([P_rows['u'].to_numpy(), cg['u'].to_numpy()]))
+            for nc in (64, 256):
+                km = MiniBatchKMeans(nc, random_state=0, batch_size=4096, n_init=3).fit(Un[act])
+                lab = km.predict(Un).astype(np.int32)
+                g = float(P_rows['clicked'].mean())
+                pr = P_rows.select('ii', 'clicked', pl.Series('cl', lab[P_rows['u'].to_numpy()]))
+                ict = pr.group_by('ii').agg(((pl.col('clicked').sum() + 20 * g) / (pl.len() + 20)).alias('ictr'))
+                cct = pr.group_by('cl', 'ii').agg(pl.col('clicked').sum().alias('c'), pl.len().alias('n'))
+                q = pl.DataFrame({'cl': lab[uu], 'ii': ii.astype(np.int32), 'r': np.arange(len(uu))}).join(cct, on=['cl', 'ii'], how='left') \
+                      .join(ict, on='ii', how='left').sort('r').with_columns(pl.col('ictr').fill_null(g), pl.col('c').fill_null(0), pl.col('n').fill_null(0))
+                F[f'x_clctr{nc}'] = ((q['c'] + 10 * q['ictr']) / (q['n'] + 10)).to_numpy()
+                F[f'x_clctr{nc}_lift'] = F[f'x_clctr{nc}'] / q['ictr'].to_numpy()
+                F[f'x_climp{nc}'] = q['n'].to_numpy().astype(np.float32)
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])
+    zc = [c for c in F if not c.startswith('x_climp')]
+    R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in zc] +
+                       [pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in zc])
+    return R.drop('eid', 'u', 'ii')
+for name in sys.argv[1:]:
+    t0 = time.time(); C, W = SPEC[name]
+    if W == 'eval': R = build(C, EE, ER)
+    else:
+        T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True)

parallel/src_snapshot/v2work/extra2.py

@@ -0,0 +1,73 @@
+"""Extra features #2 (leak-free; everything from ts < C):
+ - ridge 'taste vector' per user on frozen SVD k64 item vectors (clicks y=1 incl. pre-C click log, skipped impressions y=0 or -0.5)
+ - user state: time since last event/click, event-level click rate, recent-event click streaks, events in last 24h before C
+Usage: python extra2.py NAME... -> v2work/feat_x2/NAME.parquet"""
+import sys, os, time, numpy as np, polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+sys.path.insert(0, '/workspace/huawei/v3')
+from features import load, click_log
+from split import J20, DAY, first_events
+OUT = '/workspace/huawei/v2work/feat_x2'; os.makedirs(OUT, exist_ok=True)
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None),
+        'val22': (J20 + 2 * DAY, None), 'eval': (J20 + 7 * DAY, 'eval'),
+        **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}}
+def nrm(M): return M / (np.linalg.norm(M, axis=1, keepdims=True) + 1e-9)
+def ridge_users(Wm, Ym, V, lam):
+    """per-user solve (V_I^T W V_I + lam I) w = V_I^T (W*y). Wm, Ym: csr users x items (weights, weight*target)."""
+    k = V.shape[1]; A = np.zeros((Wm.shape[0], k, k), np.float32)
+    for a in range(k):
+        cols = Wm @ (V * V[:, a:a + 1])          # (NU, k): sum_i w_ui v_ia v_ib
+        A[:, a, :] = cols
+    A += lam * np.eye(k, dtype=np.float32)[None]
+    b = np.asarray(Ym @ V, dtype=np.float32)
+    return np.linalg.solve(A, b[:, :, None])[:, :, 0]
+def build(C, T_ev, T_rows):
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    CL = click_log(C, P_ev, P_rows, T_ev)            # user-level: train data + each target row's own history
+    CLg = click_log(C, P_ev, P_rows, T_ev.head(0))   # global/item-level: train data only
+    assert CL['ts'].max() < C and P_rows['ts'].max() < C
+    R = T_rows.join(T_ev.select('eid', 'u', 'ts'), on='eid', how='inner').select('rowid', 'eid', 'u', 'ii', 'ts')
+    uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+    # SVD exactly as v3 (tf-idf, row-normalised user x item click matrix)
+    cnt = CLg.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); Xc = sparse.diags(1 / (np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6)) @ Xc
+    U, S, Vt = randomized_svd(Xc, 64, random_state=0, n_iter=5); Vn = nrm(Vt.T * np.sqrt(S)).astype(np.float32)
+    # (user,item) labels: clicked anywhere before C -> 1 ; shown-but-never-clicked -> 0
+    pos = CL.select('u', 'ii').unique().with_columns(pl.lit(1.0).alias('y'))
+    neg = P_rows.filter(pl.col('clicked') == 0).select('u', 'ii').unique().join(pos.select('u', 'ii'), on=['u', 'ii'], how='anti').with_columns(pl.lit(0.0).alias('y'))
+    L = pl.concat([pos, neg])
+    lu = L['u'].to_numpy(); li = L['ii'].to_numpy(); ly = L['y'].to_numpy().astype(np.float32)
+    Wm = sparse.csr_matrix((np.ones(len(ly), np.float32), (lu, li)), shape=(NU, NI))
+    cand = Vn[ii]; F = {}
+    for lam, yneg, nm in ((1.0, 0.0, 'l1'), (10.0, 0.0, 'l10'), (3.0, -0.5, 'l3n')):
+        yy = np.where(ly > 0, 1.0, yneg).astype(np.float32)
+        Ym = sparse.csr_matrix((yy, (lu, li)), shape=(NU, NI))
+        Wu = ridge_users(Wm, Ym, Vn, lam)
+        F[f'r_{nm}'] = (Wu[uu] * cand).sum(1)
+    # user state features
+    ue = P_ev.select('eid', 'u', 'ts').join(P_rows.group_by('eid').agg(pl.col('clicked').sum().alias('nc')), on='eid', how='left') \
+             .with_columns(pl.col('nc').fill_null(0)).sort('u', 'ts')
+    us = ue.group_by('u').agg(pl.col('ts').last().alias('last_ev'), (pl.col('nc') > 0).mean().alias('s_evrate'), pl.len().alias('s_nev'),
+                              pl.col('nc').tail(1).sum().alias('s_clk_last1'), pl.col('nc').tail(3).sum().alias('s_clk_last3'),
+                              (pl.col('nc').tail(5) > 0).sum().alias('s_evclk_last5'), (pl.col('ts') >= C - DAY).sum().alias('s_nev24h'),
+                              pl.col('ts').filter(pl.col('nc') > 0).max().alias('last_clkev'))
+    lc = CL.group_by('u').agg(pl.col('ts').max().alias('last_click'), (pl.col('ts') >= C - DAY).sum().alias('s_clk24h'))
+    E = T_ev.select('eid', 'u', 'ts').join(us, on='u', how='left').join(lc, on='u', how='left').with_columns(
+        ((pl.col('ts') - pl.col('last_ev')) / 3600).alias('s_since_ev_h'), ((pl.col('ts') - pl.col('last_click')) / 3600).alias('s_since_clk_h'),
+        ((pl.col('ts') - pl.col('last_clkev')) / 3600).alias('s_since_clkev_h')).drop('u', 'ts', 'last_ev', 'last_click', 'last_clkev')
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in F.items()])
+    R = R.with_columns([((pl.col(c) - pl.col(c).mean().over('eid')) / (pl.col(c).std().over('eid') + 1e-6)).alias(c + '_z') for c in F] +
+                       [(pl.col(c) - pl.col(c).max().over('eid')).alias(c + '_dmax') for c in F])
+    R = R.join(E, on='eid', how='left')
+    return R.drop('eid', 'u', 'ii', 'ts')
+for name in sys.argv[1:]:
+    t0 = time.time(); C, W = SPEC[name]
+    if W == 'eval': R = build(C, EE, ER)
+    else:
+        T_ev = first_events(TE, C, W); R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'))
+    R.write_parquet(f'{OUT}/{name}.parquet'); print(name, R.shape, f'{time.time()-t0:.0f}s', flush=True)

parallel/src_snapshot/v2work/feats.py

@@ -0,0 +1,225 @@
+"""Feature builder for TechArena CTR. All stats for a cutoff C use only events with ts < C
+(plus the target events' own histories, which by construction contain only clicks < their ts)."""
+import polars as pl, numpy as np, time
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+
+DAY = 86400
+K_SMOOTH = 20
+
+def log(*a):
+    print(time.strftime('%H:%M:%S'), *a, flush=True)
+
+def load(D):
+    TE = pl.read_parquet(f'{D}/train_events.parquet')
+    TR = pl.read_parquet(f'{D}/train_rows.parquet')
+    EE = pl.read_parquet(f'{D}/evaluation_events.parquet')
+    ER = pl.read_parquet(f'{D}/evaluation_rows.parquet')
+    # global item index
+    items = pl.concat([TR['item'], ER['item'], TE['h_items'].explode().drop_nulls(), EE['h_items'].explode().drop_nulls()]).unique().sort()
+    imap = pl.DataFrame({'item': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)})
+    def mapi(df, col='item'):
+        return df.join(imap.rename({'item': col, 'ii': col + '_ii'}), on=col, how='left')
+    TR = mapi(TR).drop('item').rename({'item_ii': 'ii'})
+    ER = mapi(ER).drop('item').rename({'item_ii': 'ii'})
+    def map_hist(E):
+        ex = E.select('eid', 'h_items').explode('h_items').with_row_index('r')
+        ex = ex.join(imap.rename({'item': 'h_items'}), on='h_items', how='left').sort('r')
+        h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').drop_nulls().alias('h'))
+        return E.drop('h_items').join(h, on='eid', how='left').with_columns(pl.col('h').fill_null([]))
+    TE = map_hist(TE); EE = map_hist(EE)
+    return TE, TR, EE, ER, len(items)
+
+def first_events(TE, C, W):
+    """first event per user in [C, C+W)"""
+    return (TE.filter((pl.col('ts') >= C) & (pl.col('ts') < C + W)).sort('ts')
+              .group_by('user', maintain_order=True).first())
+
+def build(C, T_ev, T_rows, TE, TR, NI, svd_k=64, with_label=True):
+    t0 = time.time()
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'user', 'ts', 'tb'), on='eid', how='inner')
+    g = P_rows['clicked'].mean() if len(P_rows) else 0.063
+    ndays = max((C - P_ev['ts'].min()) / DAY, 1e-3) if len(P_ev) else 1e-3
+
+    # ---------- click pool (user, item, ts) ----------
+    def hist_clicks(E):
+        return (E.select('user', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls()
+                 .rename({'h': 'ii', 'h_ts': 'ts'}))
+    pool = pl.concat([hist_clicks(P_ev), hist_clicks(T_ev),
+                      P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique()
+    pool = pool.with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64))
+    # global/item-level pool: train data only (no target-event histories, so nothing aggregates across eval rows);
+    # `pool` (with each target row's own history) is used only for that user's own features
+    poolg = pl.concat([hist_clicks(P_ev), P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique() \
+              .with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64))
+    log('pool', len(pool))
+
+    # ---------- item stats ----------
+    it = P_rows.group_by('ii').agg(pl.len().alias('i_imp'), pl.col('clicked').sum().alias('i_clk'),
+                                   (pl.col('ts') >= C - DAY).sum().alias('i_imp1d'),
+                                   pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().alias('i_clk1d'))
+    it = it.with_columns(((pl.col('i_clk') + K_SMOOTH * g) / (pl.col('i_imp') + K_SMOOTH)).alias('i_ctr'),
+                         ((pl.col('i_clk1d') + K_SMOOTH * g) / (pl.col('i_imp1d') + K_SMOOTH)).alias('i_ctr1d'),
+                         (pl.col('i_imp') / ndays).alias('i_imp_rate'))
+    ip = poolg.group_by('ii').agg(pl.len().alias('ip_n'), pl.col('user').n_unique().alias('ip_users'),
+                                 (pl.col('ts') >= C - DAY).sum().alias('ip_1d'),
+                                 (pl.col('ts') >= C - 3 * DAY).sum().alias('ip_3d'),
+                                 ((pl.col('ts') - C) / DAY).exp().sum().alias('ip_dec1'),
+                                 ((pl.col('ts') - C) / (3 * DAY)).exp().sum().alias('ip_dec3'),
+                                 (pl.col('ts') < 1737331200).sum().alias('ip_warm'))
+    itb = P_rows.group_by('ii', 'tb').agg(pl.len().alias('n'), pl.col('clicked').sum().alias('c'))
+    itb = itb.with_columns(((pl.col('c') + K_SMOOTH * g) / (pl.col('n') + K_SMOOTH)).alias('i_ctr_tb')).select('ii', 'tb', 'i_ctr_tb')
+
+    # ---------- user stats ----------
+    ue = P_rows.group_by('user').agg(pl.col('eid').n_unique().alias('u_ev'), pl.col('clicked').sum().alias('u_clk'),
+                                     pl.col('ts').max().alias('u_last_ev'))
+    ue = ue.with_columns(((pl.col('u_clk') + 5 * g * 10) / (pl.col('u_ev') + 5)).alias('u_ctr'))
+    up = pool.group_by('user').agg(pl.len().alias('up_n'), pl.col('ts').max().alias('up_last'),
+                                   pl.col('ii').n_unique().alias('up_uniq'),
+                                   (pl.col('ts') >= C - DAY).sum().alias('up_1d'))
+    # user-item prior impressions
+    ui = P_rows.group_by('user', 'ii').agg(pl.len().alias('ui_imp'), pl.col('clicked').sum().alias('ui_clk'),
+                                           pl.col('ts').max().alias('ui_last_imp'))
+    uip = pool.group_by('user', 'ii').agg(pl.len().alias('uip_n'), pl.col('ts').max().alias('uip_last'))
+
+    # ---------- co-occurrence (user sets) & transitions ----------
+    us = poolg.select('user', 'ii').unique()
+    ipop = us.group_by('ii').agg(pl.len().alias('pop'))
+    pairs = us.join(us, on='user', suffix='_b').filter(pl.col('ii') != pl.col('ii_b'))
+    co = pairs.group_by('ii', 'ii_b').agg(pl.len().alias('co'))
+    del pairs
+    co = (co.join(ipop, on='ii').join(ipop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b')
+            .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos'))
+            .select(pl.col('ii').cast(pl.Int32), pl.col('ii_b').cast(pl.Int32), pl.col('co').cast(pl.Int32), 'cos'))
+    log('cooc pairs', len(co))
+    seq = poolg.sort('user', 'ts')
+    tr_pairs = []
+    for lag in (1, 2, 3):
+        tr_pairs.append(seq.with_columns(pl.col('ii').shift(lag).over('user').alias('prev'))
+                           .drop_nulls('prev').select(pl.col('prev').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w')))
+    trn = pl.concat(tr_pairs).group_by('prev', 'ii').agg(pl.col('w').sum().alias('trn'))
+    trn_out = trn.group_by('prev').agg(pl.col('trn').sum().alias('trn_tot'))
+    trn = trn.join(trn_out, on='prev').with_columns((pl.col('trn') / (pl.col('trn_tot') + 5)).cast(pl.Float32).alias('trp')).select('prev', 'ii', 'trn', 'trp')
+
+    # ---------- SVD embeddings of user x item click matrix ----------
+    uidx = us.select('user').unique().with_row_index('ur')
+    m = us.join(uidx, on='user')
+    df_ = np.bincount(m['ii'].to_numpy(), minlength=NI)
+    idf = np.log((len(uidx) + 1) / (df_ + 1)).astype(np.float32)
+    X = sparse.csr_matrix((idf[m['ii'].to_numpy()], (m['ur'].to_numpy(), m['ii'].to_numpy())), shape=(len(uidx), NI))
+    rn = np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel()) + 1e-6
+    X = sparse.diags(1 / rn) @ X
+    U, S, Vt = randomized_svd(X, svd_k, random_state=0, n_iter=4)
+    V = Vt.T.astype(np.float32)  # unscaled singular vectors (best in emb_exp)
+    V /= (np.linalg.norm(V, axis=1, keepdims=True) + 1e-6)
+    V32 = V[:, :32] / (np.linalg.norm(V[:, :32], axis=1, keepdims=True) + 1e-6)
+    # fold-in of every target user's own pool (train clicks + own history) onto the train-only factors
+    uidx = pool.select('user').unique().with_row_index('ur')
+    mu = pool.select('user', 'ii').unique().join(uidx, on='user')
+    Xu = sparse.csr_matrix((idf[mu['ii'].to_numpy()], (mu['ur'].to_numpy(), mu['ii'].to_numpy())), shape=(len(uidx), NI))
+    Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu
+    U = np.asarray(Xu @ (Vt.T / S)).astype(np.float32)
+    S = np.ones_like(S)
+    log('svd done')
+
+    # ---------- assemble target rows ----------
+    R = T_rows.join(T_ev, on='eid', how='left').with_row_index('r')
+    R = R.with_columns((pl.col('ts') - C).alias('stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('hour'))
+    R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(itb, on=['ii', 'tb'], how='left')
+          .join(ue, on='user', how='left').join(up, on='user', how='left')
+          .join(ui, on=['user', 'ii'], how='left').join(uip, on=['user', 'ii'], how='left').sort('r'))
+    R = R.with_columns(
+        pl.col('h').list.len().alias('h_len'),
+        (pl.col('ts') - pl.col('h_ts').list.last()).alias('h_since_last'),
+        (pl.col('ts') - pl.col('h_ts').list.first()).alias('h_since_first'),
+        pl.col('h').list.n_unique().alias('h_uniq'),
+        pl.col('h_ts').list.eval((pl.element() >= pl.element().max() - DAY).sum()).list.first().alias('h_1d_before_last'),
+        (pl.col('ts') - pl.col('u_last_ev')).alias('u_since_ev'),
+        (pl.col('ts') - pl.col('up_last')).alias('up_since'),
+        (pl.col('ts') - pl.col('ui_last_imp')).alias('ui_since_imp'),
+        (pl.col('ts') - pl.col('uip_last')).alias('uip_since'),
+        (pl.col('ui_imp') - pl.col('ui_clk')).alias('ui_noclk'),
+        (pl.col('ip_n') / (pl.col('i_imp') + 1)).alias('ip_per_imp'),
+        (pl.col('i_ctr_tb') - pl.col('i_ctr')).alias('i_ctr_tb_diff'),
+    )
+    R = R.with_columns((pl.col('h_since_first') / (pl.col('h_len') - 1).clip(1)).alias('h_gap_mean'),
+                       (pl.col('h_uniq') / pl.col('h_len').clip(1)).alias('h_uniq_ratio'))
+
+    # pairwise candidate x history
+    PR = (R.select('r', 'ii', 'h', 'h_ts', 'ts').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos'))
+            .explode(['h', 'h_ts', 'pos']).drop_nulls('h')
+            .with_columns(pl.col('h').cast(pl.Int32)))
+    PR = PR.join(co.rename({'ii': 'ii', 'ii_b': 'h'}), on=['ii', 'h'], how='left').with_columns(pl.col('cos').fill_null(0), pl.col('co').fill_null(0))
+    PR = PR.with_columns((0.85 ** (pl.col('pos') - 1)).alias('rw'), (pl.col('h') == pl.col('ii')).alias('same'))
+    agg = PR.group_by('r').agg(
+        pl.col('cos').sum().alias('co_sum'), pl.col('cos').max().alias('co_max'), pl.col('cos').mean().alias('co_mean'),
+        (pl.col('cos') * pl.col('rw')).sum().alias('co_rw'), pl.col('co').sum().alias('co_cnt'),
+        pl.col('cos').filter(pl.col('pos') <= 3).sum().alias('co_last3'),
+        pl.col('same').sum().alias('inh_cnt'), pl.col('pos').filter(pl.col('same')).min().alias('inh_pos'),
+        (pl.col('ts') - pl.col('h_ts')).filter(pl.col('same')).min().alias('inh_since'),
+        (pl.col('cos') > 0).sum().alias('co_nz'))
+    R = R.join(agg, on='r', how='left')
+    # transitions from last 3 history items
+    TRp = (R.select('r', 'ii', pl.col('h').list.tail(3).alias('h3')).with_columns(pl.int_ranges(pl.col('h3').list.len(), 0, -1).alias('pos'))
+             .explode(['h3', 'pos']).drop_nulls('h3').rename({'h3': 'prev'}).with_columns(pl.col('prev').cast(pl.Int32))
+             .join(trn, on=['prev', 'ii'], how='left').with_columns(pl.col('trn').fill_null(0), pl.col('trp').fill_null(0)))
+    tagg = TRp.group_by('r').agg(pl.col('trn').filter(pl.col('pos') == 1).sum().alias('tr_last'),
+                                 pl.col('trp').filter(pl.col('pos') == 1).sum().alias('trp_last'),
+                                 pl.col('trn').sum().alias('tr_last3'), pl.col('trp').sum().alias('trp_last3'))
+    R = R.join(tagg, on='r', how='left').sort('r')
+    # SVD cosine: recency weighted mean of history vectors
+    hv = np.zeros((len(R), V.shape[1]), np.float32)
+    ex = PR.select('r', 'h', 'rw')
+    np.add.at(hv, ex['r'].to_numpy(), V[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+    hvn = hv / (np.linalg.norm(hv, axis=1, keepdims=True) + 1e-6)
+    cand = V[R['ii'].to_numpy()]
+    svd_cos = (hvn * cand).sum(1)
+    # user factor from SVD (pool-based, covers clicks beyond last 20)
+    uf = np.zeros((len(R), V.shape[1]), np.float32)
+    um = R.select('user').join(uidx, on='user', how='left')['ur'].to_numpy()
+    ok = ~np.isnan(um.astype(float)) if um.dtype.kind == 'f' else np.ones(len(um), bool)
+    umi = np.nan_to_num(um.astype(float), nan=0).astype(np.int64)
+    ufv = (U[umi] * np.sqrt(S)).astype(np.float32); ufv[~ok] = 0
+    ufv /= (np.linalg.norm(ufv, axis=1, keepdims=True) + 1e-6)
+    svd_ucos = (ufv * cand).sum(1)
+    hv32 = np.zeros((len(R), 32), np.float32)
+    np.add.at(hv32, ex['r'].to_numpy(), V32[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+    svd32_cos = (hv32 / (np.linalg.norm(hv32, axis=1, keepdims=True) + 1e-6) * V32[R['ii'].to_numpy()]).sum(1)
+    extra_svd = {}
+    for kk in (16, 24, 48):
+        Vk = V[:, :kk] / (np.linalg.norm(V[:, :kk], axis=1, keepdims=True) + 1e-6)
+        hk = np.zeros((len(R), kk), np.float32)
+        np.add.at(hk, ex['r'].to_numpy(), Vk[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+        extra_svd[f'svd{kk}_cos'] = (hk / (np.linalg.norm(hk, axis=1, keepdims=True) + 1e-6) * Vk[R['ii'].to_numpy()]).sum(1)
+    # pairwise svd similarity: max over history, and with the most recent item
+    pr_r = PR['r'].to_numpy(); pr_c = PR['ii'].to_numpy(); pr_h = PR['h'].to_numpy()
+    sims = np.empty(len(PR), np.float32)
+    for k in range(0, len(PR), 2_000_000):
+        sl = slice(k, k + 2_000_000); sims[sl] = (V[pr_c[sl]] * V[pr_h[sl]]).sum(1)
+    sp = pl.DataFrame({'r': pr_r, 'sim': sims, 'pos': PR['pos'].to_numpy(), 'same': PR['same'].to_numpy()}).filter(~pl.col('same'))
+    sagg = sp.group_by('r').agg(pl.col('sim').max().alias('svd_maxsim'), pl.col('sim').filter(pl.col('pos') == 1).max().alias('svd_last'),
+                                pl.col('sim').top_k(3).mean().alias('svd_top3'))
+    R = R.with_columns(pl.Series('svd_cos', svd_cos), pl.Series('svd_ucos', svd_ucos), pl.Series('svd_hnorm', np.linalg.norm(hv, axis=1)),
+                       pl.Series('svd32_cos', svd32_cos), *[pl.Series(k, v) for k, v in extra_svd.items()])
+    R = R.join(sagg, on='r', how='left').sort('r')
+
+    fill0 = ['i_imp', 'i_clk', 'i_imp1d', 'i_clk1d', 'ip_n', 'ip_users', 'ip_1d', 'ip_3d', 'ip_dec1', 'ip_dec3', 'ip_warm',
+             'u_ev', 'u_clk', 'up_n', 'up_uniq', 'up_1d', 'ui_imp', 'ui_clk', 'ui_noclk', 'uip_n', 'co_sum', 'co_max', 'co_mean', 'co_rw',
+             'co_cnt', 'co_last3', 'inh_cnt', 'co_nz', 'tr_last', 'trp_last', 'tr_last3', 'trp_last3']
+    R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])
+    R = R.with_columns(pl.col('i_ctr').fill_null(g), pl.col('i_ctr1d').fill_null(g), pl.col('i_ctr_tb').fill_null(g), pl.col('u_ctr').fill_null(g * 10))
+    # within-event relative features
+    rel = ['i_ctr', 'ip_dec1', 'ip_n', 'co_sum', 'co_rw', 'co_max', 'svd_cos', 'svd_ucos', 'svd32_cos', 'svd16_cos', 'svd24_cos', 'svd48_cos', 'svd_maxsim', 'svd_top3', 'tr_last3', 'uip_n', 'i_ctr_tb', 'ui_noclk']
+    R = R.with_columns([pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in rel] +
+                       [(pl.col(c) - pl.col(c).mean().over('eid')).alias(c + '_dm') for c in rel] +
+                       [(pl.col(c) / (pl.col(c).max().over('eid') + 1e-9)).alias(c + '_rmax') for c in ['co_sum', 'svd_cos', 'ip_dec1', 'co_rw']])
+    R = R.with_columns(pl.col('co_rw').max().over('eid').alias('ev_co_max'), pl.col('svd_cos').max().over('eid').alias('ev_svd_max'),
+                       pl.col('svd_cos').mean().over('eid').alias('ev_svd_mean'),
+                       (pl.col('inh_cnt') > 0).sum().over('eid').alias('ev_n_inh'), pl.col('i_ctr').mean().over('eid').alias('ev_ictr_mean'),
+                       pl.col('ip_dec1').mean().over('eid').alias('ev_ipdec_mean'))
+    log(f'build C={C} rows={len(R)} {time.time()-t0:.0f}s')
+    starter = dict(g=g)
+    return R
+
+DROP = ['r', 'rowid', 'eid', 'user', 'ii', 'h', 'h_ts', 'ts', 'clicked', 'u_last_ev', 'up_last', 'ui_last_imp', 'uip_last', 'fold']

parallel/src_snapshot/v2work/feats_pre_noeval.py

@@ -0,0 +1,215 @@
+"""Feature builder for TechArena CTR. All stats for a cutoff C use only events with ts < C
+(plus the target events' own histories, which by construction contain only clicks < their ts)."""
+import polars as pl, numpy as np, time
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+
+DAY = 86400
+K_SMOOTH = 20
+
+def log(*a):
+    print(time.strftime('%H:%M:%S'), *a, flush=True)
+
+def load(D):
+    TE = pl.read_parquet(f'{D}/train_events.parquet')
+    TR = pl.read_parquet(f'{D}/train_rows.parquet')
+    EE = pl.read_parquet(f'{D}/evaluation_events.parquet')
+    ER = pl.read_parquet(f'{D}/evaluation_rows.parquet')
+    # global item index
+    items = pl.concat([TR['item'], ER['item'], TE['h_items'].explode().drop_nulls(), EE['h_items'].explode().drop_nulls()]).unique().sort()
+    imap = pl.DataFrame({'item': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)})
+    def mapi(df, col='item'):
+        return df.join(imap.rename({'item': col, 'ii': col + '_ii'}), on=col, how='left')
+    TR = mapi(TR).drop('item').rename({'item_ii': 'ii'})
+    ER = mapi(ER).drop('item').rename({'item_ii': 'ii'})
+    def map_hist(E):
+        ex = E.select('eid', 'h_items').explode('h_items').with_row_index('r')
+        ex = ex.join(imap.rename({'item': 'h_items'}), on='h_items', how='left').sort('r')
+        h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').drop_nulls().alias('h'))
+        return E.drop('h_items').join(h, on='eid', how='left').with_columns(pl.col('h').fill_null([]))
+    TE = map_hist(TE); EE = map_hist(EE)
+    return TE, TR, EE, ER, len(items)
+
+def first_events(TE, C, W):
+    """first event per user in [C, C+W)"""
+    return (TE.filter((pl.col('ts') >= C) & (pl.col('ts') < C + W)).sort('ts')
+              .group_by('user', maintain_order=True).first())
+
+def build(C, T_ev, T_rows, TE, TR, NI, svd_k=64, with_label=True):
+    t0 = time.time()
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'user', 'ts', 'tb'), on='eid', how='inner')
+    g = P_rows['clicked'].mean() if len(P_rows) else 0.063
+    ndays = max((C - P_ev['ts'].min()) / DAY, 1e-3) if len(P_ev) else 1e-3
+
+    # ---------- click pool (user, item, ts) ----------
+    def hist_clicks(E):
+        return (E.select('user', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls()
+                 .rename({'h': 'ii', 'h_ts': 'ts'}))
+    pool = pl.concat([hist_clicks(P_ev), hist_clicks(T_ev),
+                      P_rows.filter(pl.col('clicked') == 1).select('user', 'ii', 'ts')]).unique()
+    pool = pool.with_columns(pl.col('ii').cast(pl.Int32), pl.col('ts').cast(pl.Int64))
+    log('pool', len(pool))
+
+    # ---------- item stats ----------
+    it = P_rows.group_by('ii').agg(pl.len().alias('i_imp'), pl.col('clicked').sum().alias('i_clk'),
+                                   (pl.col('ts') >= C - DAY).sum().alias('i_imp1d'),
+                                   pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().alias('i_clk1d'))
+    it = it.with_columns(((pl.col('i_clk') + K_SMOOTH * g) / (pl.col('i_imp') + K_SMOOTH)).alias('i_ctr'),
+                         ((pl.col('i_clk1d') + K_SMOOTH * g) / (pl.col('i_imp1d') + K_SMOOTH)).alias('i_ctr1d'),
+                         (pl.col('i_imp') / ndays).alias('i_imp_rate'))
+    ip = pool.group_by('ii').agg(pl.len().alias('ip_n'), pl.col('user').n_unique().alias('ip_users'),
+                                 (pl.col('ts') >= C - DAY).sum().alias('ip_1d'),
+                                 (pl.col('ts') >= C - 3 * DAY).sum().alias('ip_3d'),
+                                 ((pl.col('ts') - C) / DAY).exp().sum().alias('ip_dec1'),
+                                 ((pl.col('ts') - C) / (3 * DAY)).exp().sum().alias('ip_dec3'),
+                                 (pl.col('ts') < 1737331200).sum().alias('ip_warm'))
+    itb = P_rows.group_by('ii', 'tb').agg(pl.len().alias('n'), pl.col('clicked').sum().alias('c'))
+    itb = itb.with_columns(((pl.col('c') + K_SMOOTH * g) / (pl.col('n') + K_SMOOTH)).alias('i_ctr_tb')).select('ii', 'tb', 'i_ctr_tb')
+
+    # ---------- user stats ----------
+    ue = P_rows.group_by('user').agg(pl.col('eid').n_unique().alias('u_ev'), pl.col('clicked').sum().alias('u_clk'),
+                                     pl.col('ts').max().alias('u_last_ev'))
+    ue = ue.with_columns(((pl.col('u_clk') + 5 * g * 10) / (pl.col('u_ev') + 5)).alias('u_ctr'))
+    up = pool.group_by('user').agg(pl.len().alias('up_n'), pl.col('ts').max().alias('up_last'),
+                                   pl.col('ii').n_unique().alias('up_uniq'),
+                                   (pl.col('ts') >= C - DAY).sum().alias('up_1d'))
+    # user-item prior impressions
+    ui = P_rows.group_by('user', 'ii').agg(pl.len().alias('ui_imp'), pl.col('clicked').sum().alias('ui_clk'),
+                                           pl.col('ts').max().alias('ui_last_imp'))
+    uip = pool.group_by('user', 'ii').agg(pl.len().alias('uip_n'), pl.col('ts').max().alias('uip_last'))
+
+    # ---------- co-occurrence (user sets) & transitions ----------
+    us = pool.select('user', 'ii').unique()
+    ipop = us.group_by('ii').agg(pl.len().alias('pop'))
+    pairs = us.join(us, on='user', suffix='_b').filter(pl.col('ii') != pl.col('ii_b'))
+    co = pairs.group_by('ii', 'ii_b').agg(pl.len().alias('co'))
+    del pairs
+    co = (co.join(ipop, on='ii').join(ipop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b')
+            .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos'))
+            .select(pl.col('ii').cast(pl.Int32), pl.col('ii_b').cast(pl.Int32), pl.col('co').cast(pl.Int32), 'cos'))
+    log('cooc pairs', len(co))
+    seq = pool.sort('user', 'ts')
+    tr_pairs = []
+    for lag in (1, 2, 3):
+        tr_pairs.append(seq.with_columns(pl.col('ii').shift(lag).over('user').alias('prev'))
+                           .drop_nulls('prev').select(pl.col('prev').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w')))
+    trn = pl.concat(tr_pairs).group_by('prev', 'ii').agg(pl.col('w').sum().alias('trn'))
+    trn_out = trn.group_by('prev').agg(pl.col('trn').sum().alias('trn_tot'))
+    trn = trn.join(trn_out, on='prev').with_columns((pl.col('trn') / (pl.col('trn_tot') + 5)).cast(pl.Float32).alias('trp')).select('prev', 'ii', 'trn', 'trp')
+
+    # ---------- SVD embeddings of user x item click matrix ----------
+    uidx = us.select('user').unique().with_row_index('ur')
+    m = us.join(uidx, on='user')
+    df_ = np.bincount(m['ii'].to_numpy(), minlength=NI)
+    idf = np.log((len(uidx) + 1) / (df_ + 1)).astype(np.float32)
+    X = sparse.csr_matrix((idf[m['ii'].to_numpy()], (m['ur'].to_numpy(), m['ii'].to_numpy())), shape=(len(uidx), NI))
+    rn = np.sqrt(np.asarray(X.multiply(X).sum(1)).ravel()) + 1e-6
+    X = sparse.diags(1 / rn) @ X
+    U, S, Vt = randomized_svd(X, svd_k, random_state=0, n_iter=4)
+    V = Vt.T.astype(np.float32)  # unscaled singular vectors (best in emb_exp)
+    V /= (np.linalg.norm(V, axis=1, keepdims=True) + 1e-6)
+    V32 = V[:, :32] / (np.linalg.norm(V[:, :32], axis=1, keepdims=True) + 1e-6)
+    S = np.ones_like(S)
+    log('svd done')
+
+    # ---------- assemble target rows ----------
+    R = T_rows.join(T_ev, on='eid', how='left').with_row_index('r')
+    R = R.with_columns((pl.col('ts') - C).alias('stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('hour'))
+    R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(itb, on=['ii', 'tb'], how='left')
+          .join(ue, on='user', how='left').join(up, on='user', how='left')
+          .join(ui, on=['user', 'ii'], how='left').join(uip, on=['user', 'ii'], how='left').sort('r'))
+    R = R.with_columns(
+        pl.col('h').list.len().alias('h_len'),
+        (pl.col('ts') - pl.col('h_ts').list.last()).alias('h_since_last'),
+        (pl.col('ts') - pl.col('h_ts').list.first()).alias('h_since_first'),
+        pl.col('h').list.n_unique().alias('h_uniq'),
+        pl.col('h_ts').list.eval((pl.element() >= pl.element().max() - DAY).sum()).list.first().alias('h_1d_before_last'),
+        (pl.col('ts') - pl.col('u_last_ev')).alias('u_since_ev'),
+        (pl.col('ts') - pl.col('up_last')).alias('up_since'),
+        (pl.col('ts') - pl.col('ui_last_imp')).alias('ui_since_imp'),
+        (pl.col('ts') - pl.col('uip_last')).alias('uip_since'),
+        (pl.col('ui_imp') - pl.col('ui_clk')).alias('ui_noclk'),
+        (pl.col('ip_n') / (pl.col('i_imp') + 1)).alias('ip_per_imp'),
+        (pl.col('i_ctr_tb') - pl.col('i_ctr')).alias('i_ctr_tb_diff'),
+    )
+    R = R.with_columns((pl.col('h_since_first') / (pl.col('h_len') - 1).clip(1)).alias('h_gap_mean'),
+                       (pl.col('h_uniq') / pl.col('h_len').clip(1)).alias('h_uniq_ratio'))
+
+    # pairwise candidate x history
+    PR = (R.select('r', 'ii', 'h', 'h_ts', 'ts').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos'))
+            .explode(['h', 'h_ts', 'pos']).drop_nulls('h')
+            .with_columns(pl.col('h').cast(pl.Int32)))
+    PR = PR.join(co.rename({'ii': 'ii', 'ii_b': 'h'}), on=['ii', 'h'], how='left').with_columns(pl.col('cos').fill_null(0), pl.col('co').fill_null(0))
+    PR = PR.with_columns((0.85 ** (pl.col('pos') - 1)).alias('rw'), (pl.col('h') == pl.col('ii')).alias('same'))
+    agg = PR.group_by('r').agg(
+        pl.col('cos').sum().alias('co_sum'), pl.col('cos').max().alias('co_max'), pl.col('cos').mean().alias('co_mean'),
+        (pl.col('cos') * pl.col('rw')).sum().alias('co_rw'), pl.col('co').sum().alias('co_cnt'),
+        pl.col('cos').filter(pl.col('pos') <= 3).sum().alias('co_last3'),
+        pl.col('same').sum().alias('inh_cnt'), pl.col('pos').filter(pl.col('same')).min().alias('inh_pos'),
+        (pl.col('ts') - pl.col('h_ts')).filter(pl.col('same')).min().alias('inh_since'),
+        (pl.col('cos') > 0).sum().alias('co_nz'))
+    R = R.join(agg, on='r', how='left')
+    # transitions from last 3 history items
+    TRp = (R.select('r', 'ii', pl.col('h').list.tail(3).alias('h3')).with_columns(pl.int_ranges(pl.col('h3').list.len(), 0, -1).alias('pos'))
+             .explode(['h3', 'pos']).drop_nulls('h3').rename({'h3': 'prev'}).with_columns(pl.col('prev').cast(pl.Int32))
+             .join(trn, on=['prev', 'ii'], how='left').with_columns(pl.col('trn').fill_null(0), pl.col('trp').fill_null(0)))
+    tagg = TRp.group_by('r').agg(pl.col('trn').filter(pl.col('pos') == 1).sum().alias('tr_last'),
+                                 pl.col('trp').filter(pl.col('pos') == 1).sum().alias('trp_last'),
+                                 pl.col('trn').sum().alias('tr_last3'), pl.col('trp').sum().alias('trp_last3'))
+    R = R.join(tagg, on='r', how='left').sort('r')
+    # SVD cosine: recency weighted mean of history vectors
+    hv = np.zeros((len(R), V.shape[1]), np.float32)
+    ex = PR.select('r', 'h', 'rw')
+    np.add.at(hv, ex['r'].to_numpy(), V[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+    hvn = hv / (np.linalg.norm(hv, axis=1, keepdims=True) + 1e-6)
+    cand = V[R['ii'].to_numpy()]
+    svd_cos = (hvn * cand).sum(1)
+    # user factor from SVD (pool-based, covers clicks beyond last 20)
+    uf = np.zeros((len(R), V.shape[1]), np.float32)
+    um = R.select('user').join(uidx, on='user', how='left')['ur'].to_numpy()
+    ok = ~np.isnan(um.astype(float)) if um.dtype.kind == 'f' else np.ones(len(um), bool)
+    umi = np.nan_to_num(um.astype(float), nan=0).astype(np.int64)
+    ufv = (U[umi] * np.sqrt(S)).astype(np.float32); ufv[~ok] = 0
+    ufv /= (np.linalg.norm(ufv, axis=1, keepdims=True) + 1e-6)
+    svd_ucos = (ufv * cand).sum(1)
+    hv32 = np.zeros((len(R), 32), np.float32)
+    np.add.at(hv32, ex['r'].to_numpy(), V32[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+    svd32_cos = (hv32 / (np.linalg.norm(hv32, axis=1, keepdims=True) + 1e-6) * V32[R['ii'].to_numpy()]).sum(1)
+    extra_svd = {}
+    for kk in (16, 24, 48):
+        Vk = V[:, :kk] / (np.linalg.norm(V[:, :kk], axis=1, keepdims=True) + 1e-6)
+        hk = np.zeros((len(R), kk), np.float32)
+        np.add.at(hk, ex['r'].to_numpy(), Vk[ex['h'].to_numpy()] * ex['rw'].to_numpy()[:, None].astype(np.float32))
+        extra_svd[f'svd{kk}_cos'] = (hk / (np.linalg.norm(hk, axis=1, keepdims=True) + 1e-6) * Vk[R['ii'].to_numpy()]).sum(1)
+    # pairwise svd similarity: max over history, and with the most recent item
+    pr_r = PR['r'].to_numpy(); pr_c = PR['ii'].to_numpy(); pr_h = PR['h'].to_numpy()
+    sims = np.empty(len(PR), np.float32)
+    for k in range(0, len(PR), 2_000_000):
+        sl = slice(k, k + 2_000_000); sims[sl] = (V[pr_c[sl]] * V[pr_h[sl]]).sum(1)
+    sp = pl.DataFrame({'r': pr_r, 'sim': sims, 'pos': PR['pos'].to_numpy(), 'same': PR['same'].to_numpy()}).filter(~pl.col('same'))
+    sagg = sp.group_by('r').agg(pl.col('sim').max().alias('svd_maxsim'), pl.col('sim').filter(pl.col('pos') == 1).max().alias('svd_last'),
+                                pl.col('sim').top_k(3).mean().alias('svd_top3'))
+    R = R.with_columns(pl.Series('svd_cos', svd_cos), pl.Series('svd_ucos', svd_ucos), pl.Series('svd_hnorm', np.linalg.norm(hv, axis=1)),
+                       pl.Series('svd32_cos', svd32_cos), *[pl.Series(k, v) for k, v in extra_svd.items()])
+    R = R.join(sagg, on='r', how='left').sort('r')
+
+    fill0 = ['i_imp', 'i_clk', 'i_imp1d', 'i_clk1d', 'ip_n', 'ip_users', 'ip_1d', 'ip_3d', 'ip_dec1', 'ip_dec3', 'ip_warm',
+             'u_ev', 'u_clk', 'up_n', 'up_uniq', 'up_1d', 'ui_imp', 'ui_clk', 'ui_noclk', 'uip_n', 'co_sum', 'co_max', 'co_mean', 'co_rw',
+             'co_cnt', 'co_last3', 'inh_cnt', 'co_nz', 'tr_last', 'trp_last', 'tr_last3', 'trp_last3']
+    R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])
+    R = R.with_columns(pl.col('i_ctr').fill_null(g), pl.col('i_ctr1d').fill_null(g), pl.col('i_ctr_tb').fill_null(g), pl.col('u_ctr').fill_null(g * 10))
+    # within-event relative features
+    rel = ['i_ctr', 'ip_dec1', 'ip_n', 'co_sum', 'co_rw', 'co_max', 'svd_cos', 'svd_ucos', 'svd32_cos', 'svd16_cos', 'svd24_cos', 'svd48_cos', 'svd_maxsim', 'svd_top3', 'tr_last3', 'uip_n', 'i_ctr_tb', 'ui_noclk']
+    R = R.with_columns([pl.col(c).rank('average').over('eid').alias(c + '_rk') for c in rel] +
+                       [(pl.col(c) - pl.col(c).mean().over('eid')).alias(c + '_dm') for c in rel] +
+                       [(pl.col(c) / (pl.col(c).max().over('eid') + 1e-9)).alias(c + '_rmax') for c in ['co_sum', 'svd_cos', 'ip_dec1', 'co_rw']])
+    R = R.with_columns(pl.col('co_rw').max().over('eid').alias('ev_co_max'), pl.col('svd_cos').max().over('eid').alias('ev_svd_max'),
+                       pl.col('svd_cos').mean().over('eid').alias('ev_svd_mean'),
+                       (pl.col('inh_cnt') > 0).sum().over('eid').alias('ev_n_inh'), pl.col('i_ctr').mean().over('eid').alias('ev_ictr_mean'),
+                       pl.col('ip_dec1').mean().over('eid').alias('ev_ipdec_mean'))
+    log(f'build C={C} rows={len(R)} {time.time()-t0:.0f}s')
+    starter = dict(g=g)
+    return R
+
+DROP = ['r', 'rowid', 'eid', 'user', 'ii', 'h', 'h_ts', 'ts', 'clicked', 'u_last_ev', 'up_last', 'ui_last_imp', 'uip_last', 'fold']

parallel/src_snapshot/v2work/prep.py

@@ -0,0 +1,22 @@
+import polars as pl, time, sys
+D='/workspace/huawei/v2work'
+def hexid(e, n=15):  # opaque id -> int64 via first 15 hex chars after prefix
+    return e.str.slice(4, n).str.to_integer(base=16)
+def prep(name, has_label):
+    t=time.time()
+    lf=pl.scan_csv(f'{D}/{name}.csv', schema_overrides={'history_item_ids':pl.Utf8,'history_timestamps':pl.Utf8})
+    cols=[pl.col('rowid'), hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('item_id')).alias('item')]
+    if has_label: cols.append(pl.col('clicked').cast(pl.Int8))
+    lf.select(cols).sink_parquet(f'{D}/{name}_rows.parquet')
+    print(name,'rows done',time.time()-t,flush=True)
+    ev=(lf.select('event_id','user_id','timestamp','time_bucket','day_type','history_item_ids','history_timestamps')
+          .unique(subset='event_id',keep='any')
+          .select(hexid(pl.col('event_id')).alias('eid'), hexid(pl.col('user_id')).alias('user'),
+                  pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'),
+                  pl.col('day_type').cast(pl.Int8).alias('dt'),
+                  pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).list.eval(pl.element().str.slice(4,15).str.to_integer(base=16)).alias('h_items'),
+                  pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts')))
+    ev.collect(engine='streaming').sort('ts').write_parquet(f'{D}/{name}_events.parquet')
+    print(name,'events done',time.time()-t,flush=True)
+prep('evaluation',False)
+prep('train',True)

parallel/src_snapshot/v2work/starter_split.py

@@ -0,0 +1,12 @@
+import csv, polars as pl, sys
+csv.field_size_limit(10**9)
+sys.path.insert(0,'/workspace/huawei/v2work')
+D='/workspace/huawei/v2work'; C=1737331200+4*86400
+te=pl.read_parquet(f'{D}/train_events.parquet',columns=['eid','user','ts'])
+v=set(te.filter(pl.col('ts')>=C).sort('ts').group_by('user',maintain_order=True).first()['eid'].to_list())
+with open(f'{D}/train.csv',newline='') as f, open(f'{D}/st_fit.csv','w',newline='') as a, open(f'{D}/st_val.csv','w',newline='') as b, open(f'{D}/st_val_labels.csv','w',newline='') as c:
+    r=csv.reader(f); h=next(r); wa=csv.writer(a); wb=csv.writer(b); wc=csv.writer(c)
+    wa.writerow(h); wb.writerow(h[:-1]); wc.writerow(['rowid','clicked'])
+    for row in r:
+        if int(row[4])<C: wa.writerow(row)
+        elif int(row[1][4:19],16) in v: wb.writerow(row[:-1]); wc.writerow([row[0],row[-1]])

parallel/src_snapshot/v2work/starter_val.py

@@ -0,0 +1,29 @@
+import polars as pl, numpy as np, math, sys
+sys.path.insert(0,'/workspace/huawei/v2work')
+from feats import load, first_events
+from sklearn.metrics import roc_auc_score
+D='/workspace/huawei/v2work'
+TE,TR,EE,ER,NI=load(D)
+C=1737763200; W=2*86400
+T_ev=first_events(TE,C,W); T_rows=TR.join(T_ev.select('eid'),on='eid')
+P_ev=TE.filter(pl.col('ts')<C); P=TR.join(P_ev.select('eid','user'),on='eid')
+g=P['clicked'].mean(); K=50
+ic=P.group_by('ii').agg(((pl.col('clicked').sum()+K*g)/(pl.len()+K)).alias('ictr'))
+uc=P.group_by('user').agg(((pl.col('clicked').sum()+K*g)/(pl.len()+K)).alias('uctr'))
+# user_interacted: union of history items of train rows' events + clicked items
+inter=pl.concat([P_ev.select('user','h').explode('h').drop_nulls().rename({'h':'ii'}).with_columns(pl.col('ii').cast(pl.Int32)),
+                 P.filter(pl.col('clicked')==1).select('user',pl.col('ii').cast(pl.Int32))]).unique()
+pop=inter.group_by('ii').agg(pl.len().alias('pop'))
+pairs=inter.join(inter,on='user',suffix='_b').filter(pl.col('ii')!=pl.col('ii_b')).group_by('ii','ii_b').agg(pl.len().alias('c'))
+pairs=pairs.join(pop,on='ii').join(pop.rename({'ii':'ii_b','pop':'pop_b'}),on='ii_b').with_columns((pl.col('c')/(pl.col('pop')*pl.col('pop_b')).sqrt()).alias('cs')).select('ii','ii_b','cs')
+R=T_rows.join(T_ev.select('eid','user','h'),on='eid').with_row_index('r')
+PR=R.select('r',pl.col('ii').cast(pl.Int32),'h').explode('h').drop_nulls('h').rename({'h':'ii_b'}).with_columns(pl.col('ii_b').cast(pl.Int32))
+cf=PR.join(pairs,on=['ii','ii_b'],how='left').group_by('r').agg(pl.col('cs').fill_null(0).sum().alias('cf'))
+R=R.join(ic,on='ii',how='left').join(uc,on='user',how='left').join(cf,on='r',how='left').with_columns(pl.col('ictr').fill_null(g),pl.col('uctr').fill_null(g),pl.col('cf').fill_null(0))
+lg=lambda p: np.log(np.clip(p,1e-6,1-1e-6)/(1-np.clip(p,1e-6,1-1e-6)))
+z=lg(R['ictr'].to_numpy())+lg(R['uctr'].to_numpy())-lg(g)+25*R['cf'].to_numpy()
+y=R['clicked'].to_numpy()
+print('val events',len(T_ev),'rows',len(R),'ctr',y.mean())
+print('starter AUC',roc_auc_score(y,z))
+print('item ctr only',roc_auc_score(y,R['ictr'].to_numpy()),'user ctr only',roc_auc_score(y,R['uctr'].to_numpy()),'cf only',roc_auc_score(y,R['cf'].to_numpy()))
+print('no cf',roc_auc_score(y,lg(R['ictr'].to_numpy())+lg(R['uctr'].to_numpy())))

parallel/src_snapshot/v2work/train_val.py

@@ -0,0 +1,39 @@
+import polars as pl, numpy as np, lightgbm as lgb, sys
+sys.path.insert(0,'/workspace/huawei/v2work')
+from feats import DROP
+from sklearn.metrics import roc_auc_score
+D='/workspace/huawei/v2work/feat'
+tr_names=sys.argv[1].split(',') if len(sys.argv)>1 else ['f1','f2','f3','f4']
+obj=sys.argv[2] if len(sys.argv)>2 else 'binary'
+tr=pl.concat([pl.read_parquet(f'{D}/{n}.parquet').with_columns(pl.lit(i).alias('fold')) for i,n in enumerate(tr_names)],how='diagonal_relaxed').sort('fold','eid','r')
+import os
+VAL=os.environ.get('VAL','val')
+va=pl.read_parquet(f'{D}/{VAL}.parquet').sort('eid','r')
+feats=[c for c in tr.columns if c not in DROP]
+excl=[x for x in (sys.argv[3].split(',') if len(sys.argv)>3 and sys.argv[3] else [])]
+feats=[f for f in feats if not any(f.startswith(e) for e in excl)]
+Xtr=tr.select(feats).to_numpy().astype(np.float32); ytr=tr['clicked'].to_numpy()
+Xva=va.select(feats).to_numpy().astype(np.float32); yva=va['clicked'].to_numpy()
+LR=float(os.environ.get('LR','0.02'));NL=int(os.environ.get('NL','31'))
+p=dict(objective=obj,learning_rate=LR,num_leaves=NL,min_data_in_leaf=500,feature_fraction=0.5,bagging_fraction=0.8,bagging_freq=1,lambda_l2=10,verbose=-1,num_threads=8,metric='auc')
+if obj=='lambdarank':
+    p.update(metric='ndcg',eval_at=[10])
+    gtr=tr.group_by(['fold','eid'],maintain_order=True).len()['len'].to_numpy(); gva=va.group_by('eid',maintain_order=True).len()['len'].to_numpy()
+    dtr=lgb.Dataset(Xtr,ytr,group=gtr,feature_name=feats); dva=lgb.Dataset(Xva,yva,group=gva,reference=dtr)
+else:
+    dtr=lgb.Dataset(Xtr,ytr,feature_name=feats); dva=lgb.Dataset(Xva,yva,reference=dtr)
+aucs=[]
+def cb(env):
+    if env.iteration%int(os.environ.get('EV','50'))==int(os.environ.get('EV','50'))-1:
+        pr=env.model.predict(Xva,num_iteration=env.iteration+1); a=roc_auc_score(yva,pr); aucs.append((env.iteration+1,a)); print(env.iteration+1,round(a,5),flush=True)
+m=lgb.train(p,dtr,num_boost_round=int(sys.argv[4]) if len(sys.argv)>4 else 1500,callbacks=[cb])
+best=max(aucs,key=lambda x:x[1]); print('BEST',obj,tr_names,excl,best)
+imp=sorted(zip(feats,m.feature_importance('gain')),key=lambda x:-x[1])
+tot=sum(v for _,v in imp)
+for f,v in imp[:30]: print(f'{f:22s} {v/tot:.4f}')
+np.save(f'/workspace/huawei/v2work/pred_{VAL}_{obj}.npy',m.predict(Xva,num_iteration=best[0]))
+pb=m.predict(Xva,num_iteration=best[0])
+def seg(name,mask):
+    if mask.sum()>0 and 0<yva[mask].sum()<mask.sum(): print(f'  seg {name:14s} rows={mask.sum():7d} auc={roc_auc_score(yva[mask],pb[mask]):.4f}')
+u_ev=va['u_ev'].to_numpy(); hl=va['h_len'].to_numpy(); dt=va['dt'].to_numpy()
+seg('all',np.ones(len(yva),bool)); seg('warm',u_ev>0); seg('hist_only',(u_ev==0)&(hl>0)); seg('empty_hist',hl==0); seg('weekday',dt==0); seg('weekend',dt==1)

parallel/src_snapshot/v2work/tt.py

@@ -0,0 +1,59 @@
+"""Two-tower click model trained on events before cutoff: user = position-weighted sum of
+history item embeddings; score = <user, item_out> + item bias. Trained with BCE per candidate
+plus an in-event softmax term (with a learned 'no click' option)."""
+import torch, numpy as np, polars as pl, time
+torch.set_num_threads(4)
+H=20
+def pack(E):
+    """events -> padded history matrix [n,H] (newest last, right-aligned), mask, plus delta-time buckets"""
+    n=len(E); M=np.full((n,H),-1,np.int64); DT=np.zeros((n,H),np.float32)
+    hl=E['h'].to_list(); htl=E['h_ts'].to_list(); ts=E['ts'].to_numpy()
+    for i,(h,t) in enumerate(zip(hl,htl)):
+        if h:
+            h=h[-H:]; t=t[-H:]; M[i,H-len(h):]=h; DT[i,H-len(h):]=np.log1p((ts[i]-np.array(t))/3600.)
+    return M,DT
+class TT(torch.nn.Module):
+    def __init__(s,NI,d=64):
+        super().__init__()
+        s.ein=torch.nn.Embedding(NI+1,d,padding_idx=NI); s.eout=torch.nn.Embedding(NI,d); s.b=torch.nn.Embedding(NI,1)
+        s.pw=torch.nn.Parameter(torch.zeros(H)); s.tw=torch.nn.Parameter(torch.zeros(1)); s.b0=torch.nn.Parameter(torch.tensor(-2.7))
+        s.null=torch.nn.Linear(d,1)
+        for e in (s.ein,s.eout): torch.nn.init.normal_(e.weight,0,0.05)
+        torch.nn.init.zeros_(s.b.weight); s.NI=NI
+    def user(s,M,DT):
+        mask=(M>=0); Mi=torch.where(mask,M,torch.full_like(M,s.NI))
+        w=torch.exp(s.pw[None,:]-torch.nn.functional.softplus(s.tw)*DT)*mask
+        u=(s.ein(Mi)*w[...,None]).sum(1)/(w.sum(1,keepdim=True)+1e-6)
+        return u
+    def forward(s,M,DT,C):
+        u=s.user(M,DT); return (u[:,None,:]*s.eout(C)).sum(-1)+s.b(C)[...,0]+s.b0, s.null(u)[:,0]
+def make_xy(E,R,with_y=True):
+    R=R.join(E.select('eid').with_row_index('ev'),on='eid').sort('ev')
+    g=R.group_by('ev',maintain_order=True).agg(pl.col('ii'),*( [pl.col('clicked')] if with_y else []))
+    C=np.array(g['ii'].to_list(),np.int64); Y=np.array(g['clicked'].to_list(),np.float32) if with_y else None
+    return C,Y,R
+def train(P_ev,P_rows,NI,epochs=3,d=64,bs=512,lr=3e-3,wd=1e-6,seed=0,log=print):
+    torch.manual_seed(seed); np.random.seed(seed)
+    M,DT=pack(P_ev); C,Y,_=make_xy(P_ev,P_rows)
+    M,DT,C,Y=map(torch.from_numpy,(M,DT,C,Y))
+    m=TT(NI,d); opt=torch.optim.Adam(m.parameters(),lr=lr)
+    n=len(M); t=time.time()
+    for ep in range(epochs):
+        perm=torch.randperm(n); tot=0
+        for k in range(0,n,bs):
+            idx=perm[k:k+bs]; s,nl=m(M[idx],DT[idx],C[idx]); y=Y[idx]
+            bce=torch.nn.functional.binary_cross_entropy_with_logits(s,y)
+            logits=torch.cat([s,nl[:,None]],1); tgt=torch.where(y.sum(1)>0,y.argmax(1),torch.full((len(y),),10))
+            ce=torch.nn.functional.cross_entropy(logits,tgt)
+            reg=wd*(m.ein.weight[M[idx].clamp(min=0)].pow(2).sum()+m.eout.weight[C[idx]].pow(2).sum())/len(idx)
+            loss=bce+0.1*ce+reg; opt.zero_grad(); loss.backward(); opt.step(); tot+=loss.item()*len(idx)
+        log(f'tt epoch {ep} loss {tot/n:.4f} {time.time()-t:.0f}s')
+    return m
+@torch.no_grad()
+def score(m,T_ev,T_rows):
+    """returns T_rows (eid,ii) joined with tt_score, tt_null, tt_unorm"""
+    m.eval(); M,DT=pack(T_ev); C,_,R=make_xy(T_ev,T_rows,with_y=False)
+    s,nl=m(torch.from_numpy(M),torch.from_numpy(DT),torch.from_numpy(C)); u=m.user(torch.from_numpy(M),torch.from_numpy(DT))
+    out=pl.DataFrame({'eid':np.repeat(T_ev['eid'].to_numpy(),10),'ii':C.ravel().astype(np.int32),'tt_score':s.numpy().ravel(),
+                      'tt_null':np.repeat(nl.numpy(),10),'tt_unorm':np.repeat(u.norm(dim=1).numpy(),10)})
+    return out

parallel/src_snapshot/v2work/tt_exp.py

@@ -0,0 +1,15 @@
+import polars as pl, numpy as np, sys, time
+sys.path.insert(0,'/workspace/huawei/v2work')
+from feats import load, first_events
+import tt
+from sklearn.metrics import roc_auc_score
+TE,TR,EE,ER,NI=load('/workspace/huawei/v2work')
+DAY=86400; C=1737331200+4*DAY
+T_ev=first_events(TE,C,3*DAY); T_rows=TR.join(T_ev.select('eid'),on='eid')
+P_ev=TE.filter(pl.col('ts')<C); P_rows=TR.join(P_ev.select('eid'),on='eid')
+ep=int(sys.argv[1]) if len(sys.argv)>1 else 3; d=int(sys.argv[2]) if len(sys.argv)>2 else 64
+m=tt.train(P_ev,P_rows,NI,epochs=ep,d=d,wd=float(sys.argv[3]) if len(sys.argv)>3 else 1e-6)
+out=tt.score(m,T_ev,T_rows).join(T_rows.select('eid',pl.col('ii').cast(pl.Int32),'clicked'),on=['eid','ii'])
+out=out.with_columns((pl.col('tt_score')-pl.col('tt_score').mean().over('eid')).alias('dm'))
+y=out['clicked'].to_numpy()
+print('TT',ep,d,'auc',roc_auc_score(y,out['tt_score']),'dm',roc_auc_score(y,out['dm']))

parallel/src_snapshot/v2work/v3feat_ne.py

@@ -0,0 +1,290 @@
+"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR.
+
+build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate.
+Every statistic uses ONLY:
+  * train events/rows with ts < C (fit window: impressions + labels), and
+  * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts).
+Since target events are each user's first event at/after C, their histories contain only clicks < C,
+so nothing observed after a target row's own event time is ever joined.
+
+Feature groups (column prefixes, used for ablation):
+  hist_  candidate vs the user's click log / own history / past impressions of that user-item pair
+  it_    item popularity & CTR (full fit window, last 24h, last 6h, trend)
+  co_    item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks
+  us_    user activity / CTR / recency / cold-start flags
+  ev_    within-event rank / z-score / event max & mean of the main features
+  tm_    time_bucket, day_type, hour, staleness since cutoff
+  sv_    truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines
+  mf_    logistic matrix factorisation score fitted on fit-window impressions
+"""
+import time
+import numpy as np
+import polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+
+J20 = 1737331200
+DAY = 86400
+G_PRIOR = 0.0629
+K_ITEM = 20          # pseudo-impressions for item CTR shrinkage
+K_USER = 5           # pseudo-events for user CTR shrinkage
+TOPK = 200           # co-click neighbours kept per item
+ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord']
+_t0 = time.time()
+
+def log(*a):
+    print(time.strftime('%H:%M:%S'), *a, flush=True)
+
+def load(D):
+    TE = pl.read_parquet(f'{D}/ev_train.parquet')
+    TR = pl.read_parquet(f'{D}/rows_train.parquet')
+    EE = pl.read_parquet(f'{D}/ev_eval.parquet')
+    ER = pl.read_parquet(f'{D}/rows_eval.parquet')
+    NI = pl.read_parquet(f'{D}/items.parquet').height
+    NU = pl.read_parquet(f'{D}/users.parquet').height
+    return TE, TR, EE, ER, NI, NU
+
+def _hist_long(E):
+    return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h')
+             .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts')))
+
+def click_log(C, P_ev, P_rows, T_ev):
+    """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories."""
+    win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts')
+    hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C)
+    CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii')
+    return CL
+
+def cooc_tables(CL, NI):
+    """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next."""
+    us = CL.select('u', 'ii').unique()
+    pop = us.group_by('ii').agg(pl.len().alias('pop'))
+    pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b'))
+               .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co')))
+    co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b')
+               .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos'))
+               .select(pl.col('ii_b').alias('j'), 'ii', 'cos')       # j = history item, ii = candidate
+               .sort('cos', descending=True).group_by('j').head(TOPK))
+    seq = CL.select('u', 'ii')  # already sorted by u, ts
+    parts = []
+    for lag in (1, 2, 3):
+        parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j')
+                        .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w')))
+    trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum())
+    tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot'))
+    trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp'))
+              .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK))
+    return co, trn
+
+def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000):
+    """Logistic MF  logit = b + bu + bi + <pu, qi>  fitted with Adagrad on fit-window impressions (numpy)."""
+    rng = np.random.default_rng(seed)
+    u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64)
+    y = P_rows['clicked'].to_numpy().astype(np.float32)
+    P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32)
+    bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32)
+    b = np.float32(np.log(y.mean() / (1 - y.mean())))
+    gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6)
+    n = len(y)
+    for ep in range(epochs):
+        perm = rng.permutation(n)
+        for s in range(0, n, batch):
+            idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx]
+            pu, qi = P[ub], Q[ib]
+            z = b + bu[ub] + bi[ib] + (pu * qi).sum(1)
+            g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32)          # dL/dz
+            m = len(idx)
+            Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m))
+            Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m))
+            dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None]
+            dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None]
+            dbu = Su @ g; dbi = Si @ g
+            gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi
+            P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ)
+            bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi)
+    return dict(b=b, bu=bu, bi=bi, P=P, Q=Q)
+
+def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'):
+    t0 = time.time()
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    g = float(P_rows['clicked'].mean())
+    ndays = (C - J20) / DAY
+    CL = click_log(C, P_ev, P_rows, T_ev)          # per-user stats: train data + each target row's OWN history
+    CLg = click_log(C, P_ev, P_rows, T_ev.head(0))  # item-level / global stats: train data only (no target-event histories)
+
+    # ---------------- item ----------------
+    def ictr(c, n):
+        return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM)
+    it = P_rows.group_by('ii').agg(
+        pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'),
+        (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'),
+        (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6'))
+    it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'),
+                         ictr('it_clk6', 'it_imp6').alias('it_ctr6'))
+    it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'),
+                         (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'),
+                         (pl.col('it_imp') / ndays).alias('it_imp_rate'))
+    ip = CLg.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'),
+                               (pl.col('ts') < J20).sum().alias('it_nclk_warm'),
+                               ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d'))
+
+    # ---------------- user ----------------
+    ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'),
+                                  pl.col('ts').max().alias('_u_last_ev'))
+    ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr'))
+    uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'),
+                              pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'),
+                              (pl.col('ts') >= C - DAY).sum().alias('us_nclk24'))
+    uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate'))
+
+    # ---------------- user x item ----------------
+    uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'),
+                                     (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk'))
+    uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'),
+                                         pl.col('ts').max().alias('_ui_last_imp'))
+    uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk'))
+
+    # ---------------- assemble ----------------
+    R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns])
+         .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r'))
+    R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left')
+          .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left'))
+    R = R.with_columns(
+        pl.col('h').list.len().alias('us_hlen'),
+        (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'),
+        pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'),
+        (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'),
+        (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'),
+        (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'),
+        (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'),
+        (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'),
+        pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt'))
+    # position (from end, 1 = most recent) of candidate in own history
+    hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos'))
+            .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii'))
+            .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own')))
+    R = R.join(hx, on='r', how='left')
+    R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY),
+                       pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY))
+    R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in'))
+
+    # ---------------- co-click ----------------
+    co, trn = cooc_tables(CLg, NI)
+    last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20'))
+    T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left')
+    X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos'])
+            .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32))
+            .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left')
+            .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0)))
+    l5 = pl.col('pos') <= 5
+    agg = X.group_by('r').agg(
+        pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'),
+        pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'),
+        pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'),
+        pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'),
+        pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'),
+        pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'),
+        (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20'))
+    R = R.join(agg, on='r', how='left').sort('r')
+
+    # ---------------- SVD embeddings ----------------
+    # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories;
+    # 'lag1d' = only clicks/impressions with ts < C - 1 day
+    if sv_mode == 'std':
+        CLsv, NEGsrc = CLg, P_rows
+    elif sv_mode == 'notgt':
+        CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'),
+                          _hist_long(P_ev).filter(pl.col('ts') < C)]).unique()
+        NEGsrc = P_rows
+    elif sv_mode == 'lag1d':
+        CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY)
+    else:
+        raise ValueError(sv_mode)
+    assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C
+    cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc
+    # user-side matrix incl. each target user's own history (fold-in onto item factors learned from train-only clicks)
+    cnt = CL.filter(pl.col('ts') < C).group_by('u', 'ii').agg(pl.len().alias('n')) if sv_mode == 'std' else cnt
+    Xu = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI)) @ sparse.diags(idf)
+    Xu = sparse.diags(1 / (np.sqrt(np.asarray(Xu.multiply(Xu).sum(1)).ravel()) + 1e-6)) @ Xu
+    uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64)
+    ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos'])
+    # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway
+    er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32)
+    er_l5 = ex['pos'].to_numpy() <= 5
+    nR = R.height
+    svcols = {}
+    for k in (16, 32, 64):
+        U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)
+        V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = np.asarray(Xu @ (Vt.T / np.sqrt(S))).astype(np.float32)  # fold-in = U*sqrt(S) for train users
+        Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9)
+        cand = Vn[iic]
+        svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1)
+        svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1)
+        hv = np.zeros((nR, k), np.float32); np.add.at(hv, er_r, Vn[er_j] * er_w[:, None])
+        hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9)
+        if k == 32:
+            svcols['sv_k32_hnorm'] = hn
+            sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1)
+            mx = np.full(nR, -1.0, np.float32); np.maximum.at(mx, er_r, sims)
+            mx5 = np.full(nR, -1.0, np.float32); np.maximum.at(mx5, er_r[er_l5], sims[er_l5])
+            svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5
+        if k == 64:
+            # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only)
+            Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+            neg = NEGsrc.filter(pl.col('clicked') == 0)
+            Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI))
+            nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9
+            nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9
+            cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1)
+            svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn
+            for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')):
+                M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9
+                svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1)
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()])
+
+    # ---------------- MF ----------------
+    if with_mf:
+        mf = fit_mf(P_rows, NU, NI)
+        uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+        dot = (mf['P'][uu] * mf['Q'][ii]).sum(1)
+        R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot),
+                           pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii]))
+
+    fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List]
+    R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])
+    R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] +
+                       [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] +
+                       [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)])
+
+    # ---------------- event context ----------------
+    main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk',
+            'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last',
+            'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else [])
+    exprs = []
+    for c in main:
+        x = pl.col(c).cast(pl.Float64)
+        exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'),
+                  ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'),
+                  x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')]
+    R = R.with_columns(exprs)
+    R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r')
+    log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s')
+    return R
+
+GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf']
+
+def feature_cols(df, groups):
+    cols = []
+    for c in df.columns:
+        if c in ID_COLS: continue
+        gname = c.split('_')[0]
+        if gname == 'ev':
+            src = c[3:].split('_')[0]  # ev_<src group>_...
+            if 'ev' in groups and src in groups: cols.append(c)
+        elif gname in groups:
+            cols.append(c)
+    return cols

parallel/src_snapshot/v2work/v3fold.py

@@ -0,0 +1,18 @@
+"""v3 features (v2work/v3feat_ne.py = v3/features.py with global/item stats from train data only) for all fold specs.
+Usage: python v3fold.py NAME... -> v2work/feat_v3/NAME.parquet
+  f1..f6: first event/user in [Ck, Ck+1d); val25/val26: [C, end of train); eval: Jan27 cutoff
+  e1..e4: [Ck, Jan25) (eval-like staleness, for cutoff-25 validation); E1..E6: [Ck, Jan27) (eval-like, final fit)"""
+import sys, os
+sys.path.insert(0, '/workspace/huawei/v2work'); sys.path.insert(1, '/workspace/huawei/v3')
+from v3feat_ne import load, build, log
+from split import J20, DAY, first_events
+TE, TR, EE, ER, NI, NU = load('/workspace/huawei/v3/data')
+OUT = '/workspace/huawei/v2work/feat_v3'; os.makedirs(OUT, exist_ok=True)
+SPEC = {**{f'f{k}': (J20 + k * DAY, DAY) for k in range(1, 7)}, 'val25': (J20 + 5 * DAY, None), 'val26': (J20 + 6 * DAY, None),
+        **{f'e{k}': (J20 + k * DAY, (5 - k) * DAY) for k in range(1, 5)}, **{f'E{k}': (J20 + k * DAY, (7 - k) * DAY) for k in range(1, 7)}}
+for n in sys.argv[1:]:
+    if n == 'eval': R = build(J20 + 7 * DAY, EE, ER, TE, TR, NI, NU)
+    else:
+        C, W = SPEC[n]; T_ev = first_events(TE, C, W)
+        R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU)
+    R.write_parquet(f'{OUT}/{n}.parquet'); log('wrote', n, R.shape)

parallel/src_snapshot/v2work/validate.py

@@ -0,0 +1,18 @@
+"""Standalone validator: python validate.py SUB.csv"""
+import sys, csv, math, numpy as np, polars as pl
+out=sys.argv[1]
+ids=pl.read_csv('/workspace/huawei/v2work/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8})['rowid'].to_list()
+raw=open(out,'rb').read(); errs=[]
+try: raw.decode('utf-8')
+except Exception: errs.append('not utf-8')
+with open(out,newline='',encoding='utf-8') as f:
+    r=csv.reader(f); h=next(r); rows=list(r)
+if h!=['rowid','predicted_probability']: errs.append(f'header {h}')
+if len(rows)!=529390: errs.append(f'rows {len(rows)}')
+sid=[x[0] for x in rows]
+if len(set(sid))!=len(sid): errs.append('dup rowids')
+if set(sid)!=set(ids): errs.append('rowid set mismatch')
+v=np.array([float(x[1]) for x in rows])
+if not (np.isfinite(v).all() and (v>=0).all() and (v<=1).all()): errs.append('bad values')
+print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.5g} max={v.max():.4g} distinct={len(np.unique(v))}')
+print('VALID' if not errs else 'INVALID '+';'.join(errs))

parallel/src_snapshot/v2work/write_sub.py

@@ -0,0 +1,27 @@
+"""Write + validate submission from eval_pred.parquet. Usage: python write_sub.py OUT.csv"""
+import sys, csv, math, hashlib, zipfile, os, numpy as np, polars as pl
+out=sys.argv[1]
+pr=pl.read_parquet(os.environ.get('PRED','/workspace/huawei/v2work/eval_pred.parquet'))
+ev=pl.read_csv('/workspace/huawei/v2work/evaluation.csv',columns=['rowid'],schema_overrides={'rowid':pl.Utf8})
+df=ev.join(pr,on='rowid',how='left',maintain_order='left')
+p=np.nan_to_num(df['p'].to_numpy().astype(np.float64),nan=0.0629,posinf=1.0,neginf=0.0).clip(0,1)
+p[np.isnan(df['p'].to_numpy().astype(float))]=0.0629
+print('missing preds filled:',int(df['p'].null_count()))
+with open(out,'w',encoding='utf-8',newline='') as f:
+    f.write('rowid,predicted_probability\n'); f.writelines(f'{r},{v:.8g}\n' for r,v in zip(df['rowid'].to_list(),p))
+# ---- validator
+ids=ev['rowid'].to_list(); errs=[]
+with open(out,newline='',encoding='utf-8') as f:
+    r=csv.reader(f); h=next(r); rows=list(r)
+if h!=['rowid','predicted_probability']: errs.append('header')
+if len(rows)!=529390 or len(ids)!=529390: errs.append(f'rows {len(rows)}')
+sid=[x[0] for x in rows]
+if len(set(sid))!=len(sid) or set(sid)!=set(ids): errs.append('rowid set mismatch')
+vals=[float(x[1]) for x in rows]
+if not all(math.isfinite(v) and 0<=v<=1 for v in vals): errs.append('bad values')
+v=np.array(vals); print(f'rows={len(rows)} unique={len(set(sid))} min={v.min():.4g} mean={v.mean():.4g} max={v.max():.4g} distinct={len(np.unique(v))}')
+print('VALID' if not errs else 'INVALID '+';'.join(errs))
+zp=out+'.zip'
+with zipfile.ZipFile(zp,'w',zipfile.ZIP_DEFLATED) as z: z.write(out,arcname=os.path.basename(out))
+with zipfile.ZipFile(zp) as z: print('zip contents',z.namelist())
+print('sha256',hashlib.sha256(open(out,'rb').read()).hexdigest())

parallel/src_snapshot/v3/ablation.py

@@ -0,0 +1,22 @@
+"""Ablation: add feature groups one at a time; binary LightGBM; val AUC at both cutoffs. Writes out/ablation.json."""
+import json, os, sys, time
+from train import load_table, fit, FOLDS
+from features import feature_cols
+D = os.path.dirname(os.path.abspath(__file__))
+LR = float(os.environ.get('ABL_LR', '0.1')); TH = int(os.environ.get('ABL_THREADS', '4'))
+steps = [['it'], ['it', 'hist'], ['it', 'hist', 'us'], ['it', 'hist', 'us', 'co'], ['it', 'hist', 'us', 'co', 'sv'],
+         ['it', 'hist', 'us', 'co', 'sv', 'mf'], ['it', 'hist', 'us', 'co', 'sv', 'mf', 'ev'],
+         ['it', 'hist', 'us', 'co', 'sv', 'mf', 'ev', 'tm']]
+out = f'{D}/out/ablation.json'
+res = json.load(open(out)) if os.path.exists(out) else {}
+for cut in (25, 26):
+    tr = load_table(FOLDS[cut]); va = load_table([f'val{cut}'])
+    for g in steps:
+        key = f'{cut}:' + '+'.join(g)
+        if key in res: continue
+        t0 = time.time()
+        feats = feature_cols(tr, g)
+        m, best, auc = fit(tr, va, feats, 'binary', LR, TH)
+        res[key] = dict(auc=auc, best_iter=best, n_feats=len(feats), secs=round(time.time() - t0))
+        print(key, res[key], flush=True)
+        json.dump(res, open(out, 'w'), indent=1)

parallel/src_snapshot/v3/baselines.py

@@ -0,0 +1,38 @@
+"""Single-score baselines on the local validation sets (val25 / val26), incl. a faithful port of starter_solution.py
+fitted only on events < cutoff. Appends to out/baselines.json."""
+import json, os, sys
+import numpy as np, polars as pl
+from features import load
+from split import C_MAIN, C_SECOND, split
+from metric import global_auc
+D = os.path.dirname(os.path.abspath(__file__))
+TE, TR, EE, ER, NI, NU = load(f'{D}/data')
+K = 50; CFW = 25.0
+lg = lambda p: np.log(np.clip(p, 1e-6, 1 - 1e-6) / (1 - np.clip(p, 1e-6, 1 - 1e-6)))
+res = {}
+for tag, C in ((25, C_MAIN), (26, C_SECOND)):
+    fe, fr, ve, vr = split(TE, TR, C)
+    P = fr.join(fe.select('eid', 'u'), on='eid')
+    g = P['clicked'].mean()
+    ic = P.group_by('ii').agg(((pl.col('clicked').sum() + K * g) / (pl.len() + K)).alias('ictr'))
+    uc = P.group_by('u').agg(((pl.col('clicked').sum() + K * g) / (pl.len() + K)).alias('uctr'))
+    inter = pl.concat([fe.select('u', 'h').explode('h').drop_nulls().rename({'h': 'ii'}).with_columns(pl.col('ii').cast(pl.Int32)),
+                       P.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32))]).unique()
+    pop = inter.group_by('ii').agg(pl.len().alias('pop'))
+    pairs = (inter.join(inter, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b')).group_by('ii', 'ii_b').agg(pl.len().alias('c'))
+             .join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b')
+             .with_columns((pl.col('c') / (pl.col('pop') * pl.col('pop_b')).sqrt()).alias('cs')).select('ii', 'ii_b', 'cs'))
+    R = vr.join(ve.select('eid', 'u', 'h'), on='eid').with_row_index('r')
+    PR = R.select('r', pl.col('ii').cast(pl.Int32), 'h').explode('h').drop_nulls('h').rename({'h': 'ii_b'}).with_columns(pl.col('ii_b').cast(pl.Int32))
+    cf = PR.join(pairs, on=['ii', 'ii_b'], how='left').group_by('r').agg(pl.col('cs').fill_null(0).sum().alias('cf'))
+    R = (R.join(ic, on='ii', how='left').join(uc, on='u', how='left').join(cf, on='r', how='left')
+          .with_columns(pl.col('ictr').fill_null(g), pl.col('uctr').fill_null(g), pl.col('cf').fill_null(0)).sort('r'))
+    y = R['clicked'].to_numpy()
+    starter = lg(R['ictr'].to_numpy()) + lg(R['uctr'].to_numpy()) - lg(g) + CFW * R['cf'].to_numpy()
+    F = pl.read_parquet(f'{D}/feat/val{tag}.parquet')
+    yf = F['clicked'].to_numpy()
+    res[tag] = dict(starter=global_auc(y, starter), starter_item_ctr_k50=global_auc(y, R['ictr'].to_numpy()),
+                    item_ctr_smoothed=global_auc(yf, F['it_ctr'].to_numpy()), item_impressions=global_auc(yf, F['it_imp'].to_numpy()),
+                    cand_in_history=global_auc(yf, F['hist_in'].to_numpy()), n_rows=len(y), n_events=ve.height, ctr=float(y.mean()))
+    print(tag, res[tag], flush=True)
+json.dump(res, open(f'{D}/out/baselines.json', 'w'), indent=1)

parallel/src_snapshot/v3/blend.py

@@ -0,0 +1,16 @@
+"""Rank-average of submission_v1 (external percentile ranks) and submission_v2. Not measurable on local validation.
+Usage: python blend.py [w_v2=0.5]"""
+import sys, numpy as np, polars as pl
+w = float(sys.argv[1]) if len(sys.argv) > 1 else 0.5
+rd = lambda p: pl.read_csv(p, schema_overrides={'rowid': pl.Utf8, 'predicted_probability': pl.Float64})
+ex = rd('/workspace/huawei/example_submission.csv').select('rowid')
+v1 = rd('/workspace/huawei/submission_v1.csv'); v2 = rd('/workspace/huawei/submission_v2.csv')
+J = ex.join(v1.rename({'predicted_probability': 'p1'}), on='rowid', how='left', maintain_order='left') \
+      .join(v2.rename({'predicted_probability': 'p2'}), on='rowid', how='left', maintain_order='left')
+assert J.height == ex.height and J['p1'].null_count() == 0 and J['p2'].null_count() == 0
+n = J.height
+r1 = J['p1'].rank('average').to_numpy() / n; r2 = J['p2'].rank('average').to_numpy() / n
+p = np.clip(np.nan_to_num((1 - w) * r1 + w * r2, nan=0.0629), 0, 1)
+out = '/workspace/huawei/v3/submission_blend.csv'
+pl.DataFrame({'rowid': J['rowid'], 'predicted_probability': p}).to_pandas().to_csv(out, index=False, float_format='%.8g')
+print('wrote', out, 'w_v2', w)

parallel/src_snapshot/v3/build_feats.py

@@ -0,0 +1,29 @@
+"""Build & cache feature tables. Usage: python build_feats.py [val|final|all]
+  folds f1..f6 : cutoff Ck=Jan20+k d, targets = first event/user in [Ck, Ck+1d)   (model-training rows)
+  val25 / val26: cutoff Jan25 / Jan26, targets = first event/user at/after cutoff (to end of train)
+  eval         : cutoff Jan27, targets = evaluation events
+"""
+import os, sys, time
+import polars as pl
+from features import load, build, log
+from split import J20, DAY, first_events
+D = os.path.dirname(os.path.abspath(__file__))
+TE, TR, EE, ER, NI, NU = load(f'{D}/data')
+mode = sys.argv[1] if len(sys.argv) > 1 else 'all'
+specs = []
+if mode in ('val', 'all'):
+    specs += [(f'f{k}', J20 + k * DAY, DAY) for k in range(1, 6)] + [('val25', J20 + 5 * DAY, None), ('val26', J20 + 6 * DAY, None)]
+if mode in ('final', 'all'):
+    specs += [s for s in [('f6', J20 + 6 * DAY, DAY), ('eval', J20 + 7 * DAY, 'eval')] if s[0] not in [x[0] for x in specs]]
+t0 = time.time()
+for name, C, W in specs:
+    out = f'{D}/feat/{name}.parquet'
+    if os.path.exists(out) and '--force' not in sys.argv:
+        log('skip', name); continue
+    if W == 'eval':
+        R = build(C, EE, ER, TE, TR, NI, NU)
+    else:
+        T_ev = first_events(TE, C, W)
+        R = build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU)
+    R.write_parquet(out)
+log(f'feature build total {time.time()-t0:.0f}s')

parallel/src_snapshot/v3/features.py

@@ -0,0 +1,285 @@
+"""Feature builder (as-of-cutoff, leakage-safe) for TechArena CTR.
+
+build(C, T_ev, T_rows, TE, TR) returns one feature row per target candidate.
+Every statistic uses ONLY:
+  * train events/rows with ts < C (fit window: impressions + labels), and
+  * click histories of events with ts < C, plus the target event's OWN history (strictly before its ts).
+Since target events are each user's first event at/after C, their histories contain only clicks < C,
+so nothing observed after a target row's own event time is ever joined.
+
+Feature groups (column prefixes, used for ablation):
+  hist_  candidate vs the user's click log / own history / past impressions of that user-item pair
+  it_    item popularity & CTR (full fit window, last 24h, last 6h, trend)
+  co_    item-item co-click (user co-occurrence cosine) and sequential transitions vs user's last 5/20 clicks
+  us_    user activity / CTR / recency / cold-start flags
+  ev_    within-event rank / z-score / event max & mean of the main features
+  tm_    time_bucket, day_type, hour, staleness since cutoff
+  sv_    truncated SVD (tf-idf, row-normalised user x item click matrix as of C): user/history/item cosines
+  mf_    logistic matrix factorisation score fitted on fit-window impressions
+"""
+import time
+import numpy as np
+import polars as pl
+from scipy import sparse
+from sklearn.utils.extmath import randomized_svd
+
+J20 = 1737331200
+DAY = 86400
+G_PRIOR = 0.0629
+K_ITEM = 20          # pseudo-impressions for item CTR shrinkage
+K_USER = 5           # pseudo-events for user CTR shrinkage
+TOPK = 200           # co-click neighbours kept per item
+ID_COLS = ['rowid', 'eid', 'u', 'ii', 'ts', 'clicked', 'ord']
+_t0 = time.time()
+
+def log(*a):
+    print(time.strftime('%H:%M:%S'), *a, flush=True)
+
+def load(D):
+    TE = pl.read_parquet(f'{D}/ev_train.parquet')
+    TR = pl.read_parquet(f'{D}/rows_train.parquet')
+    EE = pl.read_parquet(f'{D}/ev_eval.parquet')
+    ER = pl.read_parquet(f'{D}/rows_eval.parquet')
+    NI = pl.read_parquet(f'{D}/items.parquet').height
+    NU = pl.read_parquet(f'{D}/users.parquet').height
+    return TE, TR, EE, ER, NI, NU
+
+def _hist_long(E):
+    return (E.select('u', 'h', 'h_ts').explode(['h', 'h_ts']).drop_nulls('h')
+             .select('u', pl.col('h').cast(pl.Int32).alias('ii'), pl.col('h_ts').cast(pl.Int64).alias('ts')))
+
+def click_log(C, P_ev, P_rows, T_ev):
+    """Exact (user,item,ts) click log as of C: fit-window clicks + warm-up clicks seen in pre-C / own histories."""
+    win = P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts')
+    hist = pl.concat([_hist_long(P_ev), _hist_long(T_ev)]).filter(pl.col('ts') < C)
+    CL = pl.concat([win, hist]).unique().sort('u', 'ts', 'ii')
+    return CL
+
+def cooc_tables(CL, NI):
+    """User co-occurrence cosine (top-K per item) + lag-weighted transition probabilities prev->next."""
+    us = CL.select('u', 'ii').unique()
+    pop = us.group_by('ii').agg(pl.len().alias('pop'))
+    pairs = (us.join(us, on='u', suffix='_b').filter(pl.col('ii') != pl.col('ii_b'))
+               .group_by('ii', 'ii_b').agg(pl.len().cast(pl.Int32).alias('co')))
+    co = (pairs.join(pop, on='ii').join(pop.rename({'ii': 'ii_b', 'pop': 'pop_b'}), on='ii_b')
+               .with_columns((pl.col('co') / (pl.col('pop') * pl.col('pop_b')).sqrt()).cast(pl.Float32).alias('cos'))
+               .select(pl.col('ii_b').alias('j'), 'ii', 'cos')       # j = history item, ii = candidate
+               .sort('cos', descending=True).group_by('j').head(TOPK))
+    seq = CL.select('u', 'ii')  # already sorted by u, ts
+    parts = []
+    for lag in (1, 2, 3):
+        parts.append(seq.with_columns(pl.col('ii').shift(lag).over('u').alias('j')).drop_nulls('j')
+                        .select(pl.col('j').cast(pl.Int32), 'ii', pl.lit(1.0 / lag).alias('w')))
+    trn = pl.concat(parts).filter(pl.col('j') != pl.col('ii')).group_by('j', 'ii').agg(pl.col('w').sum())
+    tot = trn.group_by('j').agg(pl.col('w').sum().alias('tot'))
+    trn = (trn.join(tot, on='j').with_columns((pl.col('w') / (pl.col('tot') + 5.0)).cast(pl.Float32).alias('trp'))
+              .select('j', 'ii', 'trp').sort('trp', descending=True).group_by('j').head(TOPK))
+    return co, trn
+
+def fit_mf(P_rows, NU, NI, dim=24, epochs=6, lr=0.05, l2=1e-4, seed=0, batch=200_000):
+    """Logistic MF  logit = b + bu + bi + <pu, qi>  fitted with Adagrad on fit-window impressions (numpy)."""
+    rng = np.random.default_rng(seed)
+    u = P_rows['u'].to_numpy().astype(np.int64); i = P_rows['ii'].to_numpy().astype(np.int64)
+    y = P_rows['clicked'].to_numpy().astype(np.float32)
+    P = (rng.standard_normal((NU, dim)) * 0.01).astype(np.float32); Q = (rng.standard_normal((NI, dim)) * 0.01).astype(np.float32)
+    bu = np.zeros(NU, np.float32); bi = np.zeros(NI, np.float32)
+    b = np.float32(np.log(y.mean() / (1 - y.mean())))
+    gP = np.full_like(P, 1e-6); gQ = np.full_like(Q, 1e-6); gbu = np.full_like(bu, 1e-6); gbi = np.full_like(bi, 1e-6)
+    n = len(y)
+    for ep in range(epochs):
+        perm = rng.permutation(n)
+        for s in range(0, n, batch):
+            idx = perm[s:s + batch]; ub, ib, yb = u[idx], i[idx], y[idx]
+            pu, qi = P[ub], Q[ib]
+            z = b + bu[ub] + bi[ib] + (pu * qi).sum(1)
+            g = (1 / (1 + np.exp(-z)) - yb).astype(np.float32)          # dL/dz
+            m = len(idx)
+            Su = sparse.csr_matrix((np.ones(m, np.float32), (ub, np.arange(m))), shape=(NU, m))
+            Si = sparse.csr_matrix((np.ones(m, np.float32), (ib, np.arange(m))), shape=(NI, m))
+            dP = Su @ (g[:, None] * qi) + l2 * P * (np.asarray(Su.sum(1)).ravel() > 0)[:, None]
+            dQ = Si @ (g[:, None] * pu) + l2 * Q * (np.asarray(Si.sum(1)).ravel() > 0)[:, None]
+            dbu = Su @ g; dbi = Si @ g
+            gP += dP * dP; gQ += dQ * dQ; gbu += dbu * dbu; gbi += dbi * dbi
+            P -= lr * dP / np.sqrt(gP); Q -= lr * dQ / np.sqrt(gQ)
+            bu -= lr * dbu / np.sqrt(gbu); bi -= lr * dbi / np.sqrt(gbi)
+    return dict(b=b, bu=bu, bi=bi, P=P, Q=Q)
+
+def build(C, T_ev, T_rows, TE, TR, NI, NU, with_mf=True, sv_mode='std'):
+    t0 = time.time()
+    P_ev = TE.filter(pl.col('ts') < C)
+    P_rows = TR.join(P_ev.select('eid', 'u', 'ts'), on='eid', how='inner')
+    g = float(P_rows['clicked'].mean())
+    ndays = (C - J20) / DAY
+    CL = click_log(C, P_ev, P_rows, T_ev)
+
+    # ---------------- item ----------------
+    def ictr(c, n):
+        return (pl.col(c) + K_ITEM * g) / (pl.col(n) + K_ITEM)
+    it = P_rows.group_by('ii').agg(
+        pl.len().alias('it_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('it_clk'),
+        (pl.col('ts') >= C - DAY).sum().alias('it_imp24'), pl.col('clicked').filter(pl.col('ts') >= C - DAY).sum().cast(pl.Int32).alias('it_clk24'),
+        (pl.col('ts') >= C - 6 * 3600).sum().alias('it_imp6'), pl.col('clicked').filter(pl.col('ts') >= C - 6 * 3600).sum().cast(pl.Int32).alias('it_clk6'))
+    it = it.with_columns(ictr('it_clk', 'it_imp').alias('it_ctr'), ictr('it_clk24', 'it_imp24').alias('it_ctr24'),
+                         ictr('it_clk6', 'it_imp6').alias('it_ctr6'))
+    it = it.with_columns((pl.col('it_ctr24') / pl.col('it_ctr')).alias('it_trend24'), (pl.col('it_ctr6') / pl.col('it_ctr')).alias('it_trend6'),
+                         (pl.col('it_imp24') / (pl.col('it_imp') / ndays)).alias('it_imptrend24'),
+                         (pl.col('it_imp') / ndays).alias('it_imp_rate'))
+    ip = CL.group_by('ii').agg(pl.len().alias('it_nclk_all'), pl.col('u').n_unique().alias('it_nusers'),
+                               (pl.col('ts') < J20).sum().alias('it_nclk_warm'),
+                               ((pl.col('ts') - C) / (2 * DAY)).exp().sum().cast(pl.Float32).alias('it_clk_dec2d'))
+
+    # ---------------- user ----------------
+    ue = P_rows.group_by('u').agg(pl.col('eid').n_unique().alias('us_nev'), pl.col('clicked').sum().cast(pl.Int32).alias('us_nclk_win'),
+                                  pl.col('ts').max().alias('_u_last_ev'))
+    ue = ue.with_columns(((pl.col('us_nclk_win') + K_USER * 0.629) / (pl.col('us_nev') + K_USER)).alias('us_ctr'))
+    uc = CL.group_by('u').agg(pl.len().alias('us_nclk_all'), (pl.col('ts') < J20).sum().alias('us_nclk_warm'),
+                              pl.col('ii').n_unique().alias('us_nuniq'), pl.col('ts').max().alias('_u_last_clk'),
+                              (pl.col('ts') >= C - DAY).sum().alias('us_nclk24'))
+    uc = uc.with_columns((1 - pl.col('us_nuniq') / pl.col('us_nclk_all')).alias('us_repeat_rate'))
+
+    # ---------------- user x item ----------------
+    uic = CL.group_by('u', 'ii').agg(pl.len().alias('hist_n_all'), (pl.col('ts') < J20).sum().alias('hist_n_warm'),
+                                     (pl.col('ts') >= J20).sum().alias('hist_n_win'), pl.col('ts').max().alias('_ui_last_clk'))
+    uii = P_rows.group_by('u', 'ii').agg(pl.len().alias('hist_imp'), pl.col('clicked').sum().cast(pl.Int32).alias('_ui_clk'),
+                                         pl.col('ts').max().alias('_ui_last_imp'))
+    uii = uii.with_columns((pl.col('hist_imp') - pl.col('_ui_clk')).alias('hist_noclk'))
+
+    # ---------------- assemble ----------------
+    R = (T_rows.select([c for c in ('rowid', 'eid', 'ii', 'clicked', 'ord') if c in T_rows.columns])
+         .join(T_ev.select('eid', 'u', 'ts', 'tb', 'dt', 'h', 'h_ts'), on='eid', how='left').with_row_index('r'))
+    R = (R.join(it, on='ii', how='left').join(ip, on='ii', how='left').join(ue, on='u', how='left').join(uc, on='u', how='left')
+          .join(uic, on=['u', 'ii'], how='left').join(uii, on=['u', 'ii'], how='left'))
+    R = R.with_columns(
+        pl.col('h').list.len().alias('us_hlen'),
+        (pl.col('h').list.len() == 0).cast(pl.Int8).alias('us_nohist'),
+        pl.col('us_nev').is_null().cast(pl.Int8).alias('us_cold'),
+        (pl.col('ts') - pl.col('_u_last_ev')).alias('us_since_ev'),
+        (pl.col('ts') - pl.col('_u_last_clk')).alias('us_since_clk'),
+        (pl.col('ts') - pl.col('_ui_last_clk')).alias('hist_since_clk'),
+        (pl.col('ts') - pl.col('_ui_last_imp')).alias('hist_since_imp'),
+        (pl.col('ts') - C).alias('tm_stale'), ((pl.col('ts') // 3600) % 24).cast(pl.Int8).alias('tm_hour'),
+        pl.col('tb').alias('tm_tb'), pl.col('dt').alias('tm_dt'))
+    # position (from end, 1 = most recent) of candidate in own history
+    hx = (R.select('r', 'ii', 'h').with_columns(pl.int_ranges(pl.col('h').list.len(), 0, -1).alias('pos'))
+            .explode(['h', 'pos']).drop_nulls('h').filter(pl.col('h') == pl.col('ii'))
+            .group_by('r').agg(pl.col('pos').min().alias('hist_pos'), pl.len().alias('hist_n_own')))
+    R = R.join(hx, on='r', how='left')
+    R = R.with_columns(pl.col('hist_since_clk').fill_null(30 * DAY), pl.col('hist_since_imp').fill_null(30 * DAY),
+                       pl.col('hist_pos').fill_null(99), pl.col('hist_n_own').fill_null(0), pl.col('us_since_ev').fill_null(30 * DAY), pl.col('us_since_clk').fill_null(30 * DAY))
+    R = R.with_columns((pl.col('hist_n_all').fill_null(0) > 0).cast(pl.Int8).alias('hist_in'))
+
+    # ---------------- co-click ----------------
+    co, trn = cooc_tables(CL, NI)
+    last20 = CL.group_by('u', maintain_order=True).agg(pl.col('ii').tail(20).alias('l20'))
+    T_u = R.select('r', 'u', 'ii').join(last20, on='u', how='left')
+    X = (T_u.with_columns(pl.int_ranges(pl.col('l20').list.len(), 0, -1).alias('pos')).explode(['l20', 'pos'])
+            .drop_nulls('l20').rename({'l20': 'j'}).with_columns(pl.col('j').cast(pl.Int32))
+            .join(co, on=['j', 'ii'], how='left').join(trn, on=['j', 'ii'], how='left')
+            .with_columns(pl.col('cos').fill_null(0), pl.col('trp').fill_null(0)))
+    l5 = pl.col('pos') <= 5
+    agg = X.group_by('r').agg(
+        pl.col('cos').max().alias('co_cos_max20'), pl.col('cos').mean().alias('co_cos_mean20'),
+        pl.col('cos').filter(l5).max().alias('co_cos_max5'), pl.col('cos').filter(l5).mean().alias('co_cos_mean5'),
+        pl.col('cos').filter(pl.col('pos') == 1).first().alias('co_cos_last'),
+        pl.col('trp').max().alias('co_tr_max20'), pl.col('trp').mean().alias('co_tr_mean20'),
+        pl.col('trp').filter(l5).max().alias('co_tr_max5'), pl.col('trp').filter(l5).mean().alias('co_tr_mean5'),
+        pl.col('trp').filter(pl.col('pos') == 1).first().alias('co_tr_last'),
+        (pl.col('cos') / pl.col('pos').cast(pl.Float32).sqrt()).sum().alias('co_cos_rw'), pl.col('cos').sum().alias('co_cos_sum20'))
+    R = R.join(agg, on='r', how='left').sort('r')
+
+    # ---------------- SVD embeddings ----------------
+    # sv_mode (leakage checks only): 'std' = click log as of C; 'notgt' = drop target events' own histories;
+    # 'lag1d' = only clicks/impressions with ts < C - 1 day
+    if sv_mode == 'std':
+        CLsv, NEGsrc = CL, P_rows
+    elif sv_mode == 'notgt':
+        CLsv = pl.concat([P_rows.filter(pl.col('clicked') == 1).select('u', pl.col('ii').cast(pl.Int32), 'ts'),
+                          _hist_long(P_ev).filter(pl.col('ts') < C)]).unique()
+        NEGsrc = P_rows
+    elif sv_mode == 'lag1d':
+        CLsv, NEGsrc = CL.filter(pl.col('ts') < C - DAY), P_rows.filter(pl.col('ts') < C - DAY)
+    else:
+        raise ValueError(sv_mode)
+    assert (CLsv['ts'].max() or 0) < C and (NEGsrc['ts'].max() or 0) < C
+    cnt = CLsv.group_by('u', 'ii').agg(pl.len().alias('n'))
+    Xc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+    df_ = np.asarray(Xc.sum(0)).ravel(); idf = np.log((NU + 1) / (df_ + 1)).astype(np.float32)
+    Xc = Xc @ sparse.diags(idf); rn = np.sqrt(np.asarray(Xc.multiply(Xc).sum(1)).ravel()) + 1e-6; Xc = sparse.diags(1 / rn) @ Xc
+    uu = R['u'].to_numpy().astype(np.int64); iic = R['ii'].to_numpy().astype(np.int64)
+    ex = X.select('r', 'j', 'pos').unique(['r', 'j', 'pos'])
+    # per-event history (last 20 clicks) is shared by all candidates: take rows of one candidate per (r) anyway
+    er_r = ex['r'].to_numpy(); er_j = ex['j'].to_numpy().astype(np.int64); er_w = (1.0 / np.sqrt(ex['pos'].to_numpy())).astype(np.float32)
+    er_l5 = ex['pos'].to_numpy() <= 5
+    nR = R.height
+    svcols = {}
+    for k in (16, 32, 64):
+        U, S, Vt = randomized_svd(Xc, k, random_state=0, n_iter=5)
+        V = (Vt.T * np.sqrt(S)).astype(np.float32); Uf = (U * np.sqrt(S)).astype(np.float32)
+        Vn = V / (np.linalg.norm(V, axis=1, keepdims=True) + 1e-9); Un = Uf / (np.linalg.norm(Uf, axis=1, keepdims=True) + 1e-9)
+        cand = Vn[iic]
+        svcols[f'sv_k{k}_ucos'] = (Un[uu] * cand).sum(1)
+        svcols[f'sv_k{k}_udot'] = (Uf[uu] * V[iic]).sum(1)
+        hv = np.zeros((nR, k), np.float32); np.add.at(hv, er_r, Vn[er_j] * er_w[:, None])
+        hn = np.linalg.norm(hv, axis=1); svcols[f'sv_k{k}_hcos'] = (hv * cand).sum(1) / (hn + 1e-9)
+        if k == 32:
+            svcols['sv_k32_hnorm'] = hn
+            sims = (Vn[er_j] * Vn[iic[er_r]]).sum(1)
+            mx = np.full(nR, -1.0, np.float32); np.maximum.at(mx, er_r, sims)
+            mx5 = np.full(nR, -1.0, np.float32); np.maximum.at(mx5, er_r[er_l5], sims[er_l5])
+            svcols['sv_k32_imax20'] = mx; svcols['sv_k32_imax5'] = mx5
+        if k == 64:
+            # click centroid vs non-clicked-impression centroid (impressions frozen at C: P_rows only)
+            Sc = sparse.csr_matrix((np.ones(cnt.height, np.float32), (cnt['u'].to_numpy(), cnt['ii'].to_numpy())), shape=(NU, NI))
+            neg = NEGsrc.filter(pl.col('clicked') == 0)
+            Sn = sparse.csr_matrix((np.ones(neg.height, np.float32), (neg['u'].to_numpy(), neg['ii'].to_numpy())), shape=(NU, NI))
+            nC = np.asarray(Sc @ Vn); nC /= np.linalg.norm(nC, axis=1, keepdims=True) + 1e-9
+            nN = np.asarray(Sn @ Vn); nN /= np.linalg.norm(nN, axis=1, keepdims=True) + 1e-9
+            cc = (nC[uu] * cand).sum(1); cn = (nN[uu] * cand).sum(1)
+            svcols['sv_k64_ccos'] = cc; svcols['sv_k64_ncos'] = cn
+            for a, nm in ((0.25, '025'), (0.5, '05'), (1.0, '1')):
+                M = nC - a * nN; M /= np.linalg.norm(M, axis=1, keepdims=True) + 1e-9
+                svcols[f'sv_k64_con{nm}'] = (M[uu] * cand).sum(1)
+    R = R.with_columns([pl.Series(c, v.astype(np.float32)) for c, v in svcols.items()])
+
+    # ---------------- MF ----------------
+    if with_mf:
+        mf = fit_mf(P_rows, NU, NI)
+        uu = R['u'].to_numpy().astype(np.int64); ii = R['ii'].to_numpy().astype(np.int64)
+        dot = (mf['P'][uu] * mf['Q'][ii]).sum(1)
+        R = R.with_columns(pl.Series('mf_score', mf['b'] + mf['bu'][uu] + mf['bi'][ii] + dot),
+                           pl.Series('mf_dot', dot), pl.Series('mf_bi', mf['bi'][ii]))
+
+    fill0 = [c for c in R.columns if c.startswith(('it_imp', 'it_clk', 'it_n', 'us_n', 'hist_n', 'hist_imp', 'hist_noclk', 'co_')) and R[c].dtype != pl.List]
+    R = R.with_columns([pl.col(c).fill_null(0) for c in fill0])
+    R = R.with_columns([pl.col(c).fill_null(g) for c in ('it_ctr', 'it_ctr24', 'it_ctr6')] +
+                       [pl.col(c).fill_null(1.0) for c in ('it_trend24', 'it_trend6', 'it_imptrend24')] +
+                       [pl.col('us_ctr').fill_null(g), pl.col('it_imp_rate').fill_null(0), pl.col('us_repeat_rate').fill_null(0)])
+
+    # ---------------- event context ----------------
+    main = ['it_ctr', 'it_ctr24', 'it_imp', 'it_nclk_all', 'it_clk_dec2d', 'hist_n_all', 'hist_noclk',
+            'co_cos_max20', 'co_cos_rw', 'co_tr_max5', 'co_tr_last',
+            'sv_k16_ucos', 'sv_k32_ucos', 'sv_k64_ucos', 'sv_k32_hcos', 'sv_k64_hcos', 'sv_k32_imax20', 'sv_k64_con05', 'sv_k64_con025', 'sv_k64_ccos'] + (['mf_score', 'mf_dot'] if with_mf else [])
+    exprs = []
+    for c in main:
+        x = pl.col(c).cast(pl.Float64)
+        exprs += [x.rank('average').over('eid').alias(f'ev_{c}_rk'),
+                  ((x - x.mean().over('eid')) / (x.std().over('eid') + 1e-9)).alias(f'ev_{c}_z'),
+                  x.max().over('eid').alias(f'ev_{c}_max'), x.mean().over('eid').alias(f'ev_{c}_mean')]
+    R = R.with_columns(exprs)
+    R = R.drop([c for c in R.columns if c.startswith('_')] + ['h', 'h_ts', 'tb', 'dt']).sort('r').drop('r')
+    log(f'build C={C} rows={R.height} cols={R.width} {time.time()-t0:.0f}s')
+    return R
+
+GROUPS = ['it', 'hist', 'us', 'co', 'ev', 'tm', 'sv', 'mf']
+
+def feature_cols(df, groups):
+    cols = []
+    for c in df.columns:
+        if c in ID_COLS: continue
+        gname = c.split('_')[0]
+        if gname == 'ev':
+            src = c[3:].split('_')[0]  # ev_<src group>_...
+            if 'ev' in groups and src in groups: cols.append(c)
+        elif gname in groups:
+            cols.append(c)
+    return cols

parallel/src_snapshot/v3/leakcheck.py

@@ -0,0 +1,45 @@
+"""Leakage checks for the sv (SVD item-embedding / impression-contrast) features.
+1) asserts: every history click feeding any click log has ts < C (train folds, val, eval)
+2) rebuild f1..f4 + val25 with sv matrix variants and retrain at cutoff 25:
+   notgt = drop target events' own histories from the matrix; lag1d = matrix from clicks/impressions ts < C-1d
+Usage: python leakcheck.py [notgt|lag1d ...]"""
+import os, sys, json, time
+import numpy as np, polars as pl
+import train as T
+from features import load, build, log, feature_cols, GROUPS, _hist_long
+from split import J20, DAY, first_events
+D = os.path.dirname(os.path.abspath(__file__))
+TE, TR, EE, ER, NI, NU = load(f'{D}/data')
+res = {}
+# --- raw checks: history timestamps vs own event time and vs cutoff
+for nm, E in (('train', TE), ('eval', EE)):
+    x = E.select('eid', 'ts', 'h_ts').explode('h_ts').drop_nulls('h_ts')
+    res[f'{nm}_hist_after_event'] = int((x['h_ts'] >= x['ts']).sum())
+for C, nm in ((J20 + 5 * DAY, 'val25'), (J20 + 6 * DAY, 'val26')):
+    T_ev = first_events(TE, C)
+    h = _hist_long(T_ev)
+    res[f'{nm}_target_hist_clicks_ge_C'] = int((h['ts'] >= C).sum())
+    res[f'{nm}_target_min_ts_ge_C'] = bool(T_ev['ts'].min() >= C)
+res['eval_min_event_ts_ge_Jan27'] = bool(EE['ts'].min() >= J20 + 7 * DAY)
+res['train_max_event_ts_lt_Jan27'] = bool(TE['ts'].max() < J20 + 7 * DAY)
+res['eval_hist_clicks_ge_Jan27'] = int((_hist_long(EE)['ts'] >= J20 + 7 * DAY).sum())
+print(json.dumps(res, indent=1), flush=True)
+feats = None
+for mode in (sys.argv[1:] or ['notgt', 'lag1d']):
+    od = f'{D}/feat_leak/{mode}'; os.makedirs(od, exist_ok=True)
+    t0 = time.time()
+    for name, C, W in [(f'f{k}', J20 + k * DAY, DAY) for k in range(1, 5)] + [('val25', J20 + 5 * DAY, None)]:
+        out = f'{od}/{name}.parquet'
+        if os.path.exists(out): continue
+        T_ev = first_events(TE, C, W)
+        build(C, T_ev, TR.join(T_ev.select('eid'), on='eid', how='semi'), TE, TR, NI, NU, sv_mode=mode).write_parquet(out)
+    tr = pl.concat([pl.read_parquet(f'{od}/f{k}.parquet').with_columns(pl.lit(k, pl.Int16).alias('fold')) for k in range(1, 5)], how='diagonal_relaxed').sort('fold', 'eid')
+    va = pl.read_parquet(f'{od}/val25.parquet').with_columns(pl.lit(99, pl.Int16).alias('fold'))
+    fc = feature_cols(tr, GROUPS)
+    m, best, auc = T.fit(tr, va, fc, 'binary', 0.08, int(os.environ.get('THREADS', 4)))
+    svf = [c for c in fc if c.startswith('sv_')]
+    from metric import global_auc
+    single = {c: round(global_auc(va['clicked'].to_numpy(), va[c].fill_null(-9).to_numpy()), 4) for c in ('sv_k64_hcos', 'sv_k64_con05', 'sv_k32_ucos')}
+    r = dict(mode=mode, val25_auc=auc, best_iter=best, secs=round(time.time() - t0), single=single)
+    print(json.dumps(r), flush=True)
+    with open(f'{D}/out/leakcheck.jsonl', 'a') as f: f.write(json.dumps(r) + '\n')

parallel/src_snapshot/v3/metric.py

@@ -0,0 +1,9 @@
+"""Global ROC AUC over all rows (the competition metric; not per-event averaged)."""
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+def global_auc(y, p):
+    y = np.asarray(y); p = np.asarray(p, dtype=np.float64)
+    if y.min() == y.max():
+        return float('nan')
+    return float(roc_auc_score(y, p))

parallel/src_snapshot/v3/predict.py

@@ -0,0 +1,53 @@
+"""Final fit + prediction.
+  1. (re)build features for folds f1..f6 (cutoffs Jan21..Jan26) and eval (cutoff Jan27) if missing (--rebuild forces).
+  2. train binary LightGBM on f1..f6 for n_iter = best_iter(cutoff 26 val) * scale (more data -> a few more rounds).
+  3. predict eval, write submission (clip [0,1], NaN->0.0629, %.8g), order = example_submission.csv.
+Config read from out/best_config.json (written after validation); defaults below if missing.
+Usage: python predict.py [--rebuild] [--out /workspace/huawei/submission_v2.csv]
+"""
+import argparse, json, os, subprocess, sys, time
+import numpy as np, polars as pl
+from train import load_table, fit, FOLDS, importance
+from features import feature_cols, GROUPS
+D = os.path.dirname(os.path.abspath(__file__))
+
+def write_submission(rowid, pred, path, example='/workspace/huawei/example_submission.csv'):
+    p = np.nan_to_num(np.asarray(pred, dtype=np.float64), nan=0.0629, posinf=1.0, neginf=0.0).clip(0, 1)
+    df = pl.DataFrame({'rowid': rowid, 'predicted_probability': p})
+    ex = pl.read_csv(example, schema_overrides={'rowid': pl.Utf8}).select('rowid')
+    df = ex.join(df, on='rowid', how='left', maintain_order='left').with_columns(pl.col('predicted_probability').fill_null(0.0629))
+    with open(path, 'w', encoding='utf-8', newline='') as f:
+        f.write('rowid,predicted_probability\n')
+        f.writelines(f'{r},{v:.8g}\n' for r, v in zip(df['rowid'].to_list(), df['predicted_probability'].to_list()))
+    return df
+
+if __name__ == '__main__':
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--rebuild', action='store_true')
+    ap.add_argument('--out', default='/workspace/huawei/submission_v2.csv')
+    ap.add_argument('--threads', type=int, default=6)
+    a = ap.parse_args()
+    t0 = time.time()
+    cmd = [sys.executable, f'{D}/build_feats.py', 'all'] + (['--force'] if a.rebuild else [])
+    subprocess.run(cmd, check=True)
+    t_feat = time.time() - t0
+    cfg_path = f'{D}/out/best_config.json'
+    cfg = json.load(open(cfg_path)) if os.path.exists(cfg_path) else {}
+    groups = cfg.get('groups', GROUPS); drop = cfg.get('drop', [])
+    lr = cfg.get('lr', 0.05); n_iter = int(round(cfg.get('best_iter', 400) * cfg.get('scale', 1.15)))
+    params = cfg.get('params', None); folds = cfg.get('folds', FOLDS[27])
+    tr = load_table(folds)
+    feats = [f for f in feature_cols(tr, groups) if not any(f.startswith(d) for d in drop)]
+    print(f'final fit: {len(tr)} rows, {len(feats)} feats, lr={lr}, n_iter={n_iter}', flush=True)
+    t1 = time.time()
+    m, _, _ = fit(tr, None, feats, 'binary', lr, a.threads, max_rounds=n_iter, params=params)
+    t_fit = time.time() - t1
+    m.save_model(f'{D}/out/final_model.txt')
+    ev = pl.read_parquet(f'{D}/feat/eval.parquet')
+    pred = m.predict(ev.select(feats).to_numpy().astype(np.float32))
+    write_submission(ev['rowid'], pred, a.out)
+    info = dict(feature_build_secs=round(t_feat), fit_secs=round(t_fit), total_secs=round(time.time() - t0), n_iter=n_iter,
+                n_train_rows=len(tr), n_feats=len(feats), top15=importance(m))
+    json.dump(info, open(f'{D}/out/final_info.json', 'w'), indent=1)
+    print(json.dumps(info), flush=True)
+    subprocess.run([sys.executable, f'{D}/validate_submission.py', a.out], check=True)

parallel/src_snapshot/v3/prep.py

@@ -0,0 +1,65 @@
+"""Parse train/evaluation CSVs into compact parquet tables (dense int ids).
+
+Outputs in v3/data/:
+  rows_train.parquet  : rowid, eid, ii (item idx), clicked
+  rows_eval.parquet   : rowid, eid, ii, ord (original row order)
+  ev_train.parquet / ev_eval.parquet : eid, u (user idx), ts, tb, dt, h (list[int item idx]), h_ts (list[int])
+  items.parquet / users.parquet : id string -> idx
+Usage: python prep.py [DATA_DIR]
+"""
+import polars as pl, sys, time, os
+DATA = sys.argv[1] if len(sys.argv) > 1 else '/workspace/huawei/data'
+OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')
+os.makedirs(OUT, exist_ok=True)
+t0 = time.time()
+def log(*a): print(f'[{time.time()-t0:6.1f}s]', *a, flush=True)
+
+def scan(name):
+    return pl.scan_csv(f'{DATA}/{name}.csv', schema_overrides={'history_item_ids': pl.Utf8, 'history_timestamps': pl.Utf8,
+                       'rowid': pl.Utf8, 'event_id': pl.Utf8, 'user_id': pl.Utf8, 'item_id': pl.Utf8})
+
+raw = {}
+for name, lab in (('train', True), ('evaluation', False)):
+    lf = scan(name)
+    cols = ['rowid', 'event_id', 'item_id'] + (['clicked'] if lab else [])
+    rows = lf.select(cols).collect(engine='streaming')
+    ev = (lf.select('event_id', 'user_id', 'timestamp', 'time_bucket', 'day_type', 'history_item_ids', 'history_timestamps')
+            .unique(subset='event_id', keep='any')
+            .with_columns(pl.col('history_item_ids').str.json_decode(pl.List(pl.Utf8)).alias('h_str'),
+                          pl.col('history_timestamps').str.json_decode(pl.List(pl.Int64)).alias('h_ts'))
+            .drop('history_item_ids', 'history_timestamps').collect(engine='streaming'))
+    raw[name] = (rows, ev)
+    log(name, rows.shape, ev.shape)
+
+tr_rows, tr_ev = raw['train']; ev_rows, ev_ev = raw['evaluation']
+items = pl.concat([tr_rows['item_id'], ev_rows['item_id'], tr_ev['h_str'].explode().drop_nulls(),
+                   ev_ev['h_str'].explode().drop_nulls()]).unique().sort()
+items = pl.DataFrame({'item_id': items, 'ii': pl.arange(0, len(items), eager=True).cast(pl.Int32)})
+users = pl.concat([tr_ev['user_id'], ev_ev['user_id']]).unique().sort()
+users = pl.DataFrame({'user_id': users, 'u': pl.arange(0, len(users), eager=True).cast(pl.Int32)})
+events = pl.concat([tr_ev['event_id'], ev_ev['event_id']]).unique().sort()
+events = pl.DataFrame({'event_id': events, 'eid': pl.arange(0, len(events), eager=True).cast(pl.Int32)})
+log('items', len(items), 'users', len(users), 'events', len(events))
+items.write_parquet(f'{OUT}/items.parquet'); users.write_parquet(f'{OUT}/users.parquet')
+
+def map_rows(rows, lab):
+    r = rows.with_row_index('ord').join(events, on='event_id', how='left').join(items, on='item_id', how='left')
+    sel = ['ord', 'rowid', 'eid', 'ii'] + (['clicked'] if lab else [])
+    r = r.sort('ord').select(sel)
+    if lab: r = r.with_columns(pl.col('clicked').cast(pl.Int8))
+    return r
+
+def map_ev(ev):
+    e = ev.join(events, on='event_id', how='left').join(users, on='user_id', how='left')
+    ex = e.select('eid', 'h_str').with_columns(pl.int_ranges(0, pl.col('h_str').list.len()).alias('k')).explode(['h_str', 'k']).drop_nulls('h_str')
+    ex = ex.join(items.rename({'item_id': 'h_str'}), on='h_str', how='left').sort('eid', 'k')
+    h = ex.group_by('eid', maintain_order=True).agg(pl.col('ii').alias('h'))
+    e = e.join(h, on='eid', how='left').with_columns(pl.col('h').fill_null(pl.lit([], dtype=pl.List(pl.Int32))))
+    return e.select('eid', 'u', pl.col('timestamp').cast(pl.Int64).alias('ts'), pl.col('time_bucket').cast(pl.Int8).alias('tb'),
+                    pl.col('day_type').cast(pl.Int8).alias('dt'), 'h', 'h_ts').sort('ts', 'eid')
+
+map_rows(tr_rows, True).drop('ord').write_parquet(f'{OUT}/rows_train.parquet')
+map_rows(ev_rows, False).write_parquet(f'{OUT}/rows_eval.parquet')
+map_ev(tr_ev).write_parquet(f'{OUT}/ev_train.parquet')
+map_ev(ev_ev).write_parquet(f'{OUT}/ev_eval.parquet')
+log('done')

parallel/src_snapshot/v3/run_all.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# One-command rerun from the CSVs: parse -> features -> (optional validation) -> final fit -> submission -> validate.
+# Usage: ./run_all.sh            (final submission only, uses out/best_config.json)
+#        ./run_all.sh --validate (also baselines, ablation, val runs at cutoffs Jan25/Jan26)
+set -euo pipefail
+cd "$(dirname "$0")"
+PY=/workspace/venv/bin/python
+export POLARS_MAX_THREADS=${POLARS_MAX_THREADS:-6}
+mkdir -p data feat logs out
+$PY prep.py /workspace/huawei/data | tee logs/prep.log
+$PY build_feats.py all --force | tee logs/build_feats.log
+if [[ "${1:-}" == "--validate" ]]; then
+  $PY baselines.py | tee logs/baselines.log
+  $PY ablation.py | tee logs/ablation.log
+  $PY train.py --cutoff 25 --save full25 | tee logs/val25.log
+  $PY train.py --cutoff 26 --save full26 | tee logs/val26.log
+  $PY train.py --cutoff 25 --obj lambdarank | tee logs/lambdarank25.log
+  $PY segments.py full25 25 | tee logs/seg25.log; $PY segments.py full26 26 | tee logs/seg26.log
+  $PY leakcheck.py notgt lag1d | tee logs/leakcheck.log
+fi
+$PY predict.py --out /workspace/huawei/submission_v2.csv | tee logs/predict.log

parallel/src_snapshot/v3/segments.py

@@ -0,0 +1,17 @@
+"""Validation AUC by user segment. Usage: python segments.py full25 25"""
+import sys, os, json
+import numpy as np, polars as pl
+from train import load_table
+from metric import global_auc
+D = os.path.dirname(os.path.abspath(__file__))
+name, cut = sys.argv[1], int(sys.argv[2])
+va = load_table([f'val{cut}']); p = np.load(f'{D}/out/{name}_valpred.npy'); y = va['clicked'].to_numpy()
+cold = va['us_cold'].to_numpy() == 1; noh = va['us_nohist'].to_numpy() == 1
+segs = {'all': np.ones(len(y), bool), 'users_with_fit_rows': ~cold, 'history_only (no fit rows)': cold & ~noh,
+        'empty_history': noh, 'cand_in_history': va['hist_in'].to_numpy() == 1, 'cand_not_in_history': va['hist_in'].to_numpy() == 0}
+out = {}
+for k, m in segs.items():
+    out[k] = dict(rows=int(m.sum()), events=int(m.sum() // 10), ctr=float(y[m].mean()) if m.any() else None,
+                  auc=global_auc(y[m], p[m]) if m.any() else None)
+    print(f'{k:28s} rows={out[k]["rows"]:7d} ctr={out[k]["ctr"]:.4f} auc={out[k]["auc"]:.4f}')
+json.dump(out, open(f'{D}/out/segments_{name}.json', 'w'), indent=1)