""" model_create - ONE entry point for FINAL-AVG ("ens_e4__lp", read with rule_smooth): a METHOD that turns n labelled Amsterdam pixels per class into a classifier for every other Amsterdam pixel. from model_create import model_create model = model_create(shot_idx, shot_labels, n=None, spatial=True) labels = model.predict() # classes 1..4 for model.keep (all NON-shot rows, in order) model.proba # (25992, 4) probabilities before smoothing / the rule model.keep # the non-shot row indices predict() labels python tools/nnlab/model_create.py --n 25 --rep 0 [--no-spatial] WHAT THE METHOD IS Five members, each giving p(class | pixel) for all 25,992 Amsterdam rows: zs_xgb_dann zero-shot: (XGBoost + mean of 3 DANN seeds) / 2, Madrid-trained, no shots (cached in data/processed/zeroshot_xgb_dann_ams.npz) lgbm_upw LightGBM retrained per call on Madrid + the shots (weight 30), each city robust-z scaled by its own median/IQR (winners.train_retrained / predict_retrained) met_protoda_maha metric-learning encoder (prototypical + deep-CORAL, 400 steps, seeds 0 and 1) + within-class Mahalanobis head on the shots (metric.head_maha, prototypes shrunk to the Madrid class means) met_cos_maha same head on the cosine-classifier encoder (1500 steps, seeds 0, 1) tab_dann_rz_ft DANN MLP backbone (15 epochs, seed 0), last layer fine-tuned on the shots with an L2 pull to the Madrid weights, lam = 10 (lab.finetune_last_layer via tab_nn.head_ft) Ensemble (ensemble.py rule): log p = w0 log p_zs + (1 - w0) mean(log p of the four shot-using members), w0 = 10 / (n + 10); softmax. spatial=True (FINAL-AVG): label propagation (transductive.r_lp: Zhou et al., a = 0.8, 20 iterations, shots clamped as 10 x one-hot) on the union of a spatial r = 3 px graph and a cosine 10-NN graph on PCA-32 of robust-z features, then the lab's neighbour smoothing (r = 6, alpha = 0.5) over the non-shot rows only, then the decision rule (winners.decide: tau = 0.5 logit adjustment toward the Madrid prior + macro-F1 plug-in). spatial=False (fallback when pixel coordinates are not available, "ens_e4" + rule): ensemble -> decision rule. No graph, no smoothing. The decision rule is BATCH-LEVEL (the plug-in looks at the predicted class shares of the set it labels), so predict() labels all non-shot rows at once. WHAT IT NEEDS AT FIT TIME * the n-per-class shot rows (indices into the Amsterdam feature table) and labels; * ALL unlabelled Amsterdam features (393 columns of amsterdam_features.parquet): the per-city robust-z statistics, the CL2N centring of the metric embeddings and the kNN graph are computed from them (transductive, label-free); * spatial=True only: the pixel coordinates (px_key, py_key) for the r=3 graph and the r=6 smoothing; * the Madrid table (features + labels) - LightGBM is refitted on it each call, and the Madrid class means / covariances / prior come from it; * cached Madrid-trained encoders (NOT retrained here; a missing cache is an error): reports/nnlab/cache/metric/{protoda_s{0,1}_L400, cos_s{0,1}_L1500}.npz, reports/nnlab/cache/tab_nn/emb_dann_rz_s0.npz + selection.json (lam), data/processed/zeroshot_xgb_dann_ams.npz; the union graph is read from reports/nnlab/cache/transd_graph_ams_r3_k10_pca32.npz (rebuilt label-free if absent). Everything shot-independent is loaded lazily, once per process (STATE below). No file under reports/nnlab/preds is read, except by the CLI's comparison step. REPRODUCING THE STORED NUMBERS The lab stored every member as float32 and the ensemble read them back; the ensemble was stored as float32 before label propagation. model_create reproduces those float32 round trips (ROUND32 = True) so model.proba matches the stored files; they change the probabilities by < 1e-6 and do not matter for the method itself. LightGBM runs with n_jobs = NNLAB_THREADS (default 2 here) instead of winners' -1: that is the only intended difference from the stored lgbm_upw. ensemble.py and baselines.py are argparse scripts with side effects at import, so their few lines (the log-mean rule; the zero-shot average) are restated here; everything else is imported from lab / winners / metric / tab_nn / transductive. Amsterdam labels: model_create never reads them. The CLI draws shots with lab.shots_for and reads their labels with lab.shot_labels; score_only() (clearly separate, used only by the CLI) reads non-shot labels to compute macro F1, exactly like lab.score_method. """ from __future__ import annotations import os import sys import time from pathlib import Path os.environ.setdefault("NNLAB_THREADS", "2") sys.path.insert(0, str(Path(__file__).resolve().parent)) import lab # noqa: E402 import numpy as np # noqa: E402 W = lab.W K_ENS = 10.0 # ensemble.py --k default, as used for ens_e4 METRIC_MEMBERS = ("protoda", "cos") # met_protoda_maha, met_cos_maha TAB_BACKBONE, TAB_INPUT, TAB_SEEDS = "dann", "rz", (0,) GRAPH_TAG = "ams_r3_k10_pca32" ROUND32 = True # mimic the lab's float32 prediction store def _r32(P): P = np.asarray(P, np.float64) return P.astype(np.float32).astype(np.float64) if ROUND32 else P # --------------------------------------------------------------------------- # # Shot-independent state, loaded once per process # --------------------------------------------------------------------------- # class _State: def __init__(self): self._c = {} def get(self, key, fn): if key not in self._c: t = time.time() self._c[key] = fn() if os.environ.get("MC_VERBOSE"): print(f" [model_create] loaded {key} in {time.time() - t:.1f}s", flush=True) return self._c[key] # labels-free tables ------------------------------------------------------ # def data(self): """lab.load (float32 features, coords, Madrid labels/prior).""" return self.get("data", lambda: lab.load(official60=False)) def X64(self): """Madrid / Amsterdam features as float64, exactly as winners.load_data builds them (parquet -> to_numpy(float)) but without reading Amsterdam age_class.""" def f(): import pandas as pd feats = W.feature_names() Xm = pd.read_parquet(W.DATA / "madrid_features.parquet", columns=feats)[feats].to_numpy(float) Xa = pd.read_parquet(W.DATA / "amsterdam_features.parquet", columns=feats)[feats].to_numpy(float) return Xm, Xa return self.get("X64", f) def zero_shot(self): """zs_xgb_dann exactly as baselines.py builds it.""" def f(): z = np.load(lab.PROC / "zeroshot_xgb_dann_ams.npz") assert (z["pa"] == self.data()["pa"]).all() return _r32((z["P_xgb"] + z["P_dann_seeds"].mean(0)) / 2) return self.get("zs", f) # metric encoders ---------------------------------------------------------- # def metric(self): """Import tools/nnlab/metric.py (it parses argv and loads data at import).""" def f(): argv = sys.argv sys.argv = [str(lab.HERE / "metric.py")] try: import metric as M finally: sys.argv = argv return M return self.get("metric_module", f) def metric_prep(self, enc): """Per seed: CL2N Amsterdam embeddings, Madrid class means and within-class covariance - exactly metric.main()'s `prep`, from the cached encoders.""" def f(): M = self.metric() sel_p = M.MC / f"{enc}_select.json" assert sel_p.exists(), f"missing {sel_p} (would re-run the length selection)" L = M.select_length(enc)["chosen"] prep = [] for s in M.SEEDS: p = M.MC / f"{enc}_s{s}_L{L}.npz" assert p.exists(), f"missing encoder cache {p} (model_create never retrains)" c = M.encoder_cache(enc, s, L) Zm, Za = c["Hm"] @ c["W"].T + c["b"], c["Ha"] @ c["W"].T + c["b"] Em, Ea = M.cl2n(Zm), M.cl2n(Za) prep.append(dict(Ea=Ea, mu=M.class_means(Em, M.ym), Sm=M.within_cov(Em, M.ym))) return prep return self.get(f"metric_{enc}", f) # tab_nn backbone ------------------------------------------------------------ # def tab(self): def f(): import json import tab_nn as T cache = lab.CACHE / "tab_nn" sel = json.loads((cache / "selection.json").read_text()) embs = [] for s in TAB_SEEDS: p = cache / f"emb_{TAB_BACKBONE}_{TAB_INPUT}_s{s}.npz" assert p.exists(), f"missing backbone cache {p} (model_create never retrains)" z = np.load(p) assert int(z["epochs"]) == int(sel["epochs"][TAB_BACKBONE]), "stale backbone cache" embs.append(dict(Zm=z["Zm"], Za=z["Za"], W0=z["W0"], b0=z["b0"], ym=self.data()["ym"])) return T, embs, float(sel["lam"]) return self.get("tab", f) # graph ---------------------------------------------------------------------- # def graph(self): def f(): import transductive as TR d = self.data() return TR, TR.Graph(TR.union_graph(d["Xa"], d["pa"], tag=GRAPH_TAG), d["pa"]) return self.get("graph", f) STATE = _State() # --------------------------------------------------------------------------- # # Members (each: shots -> (25992, 4) probabilities) # --------------------------------------------------------------------------- # def member_lgbm_upw(shot, ys): Xm, Xa = STATE.X64() ym = STATE.data()["ym"] saved = dict(W.RETRAIN_OVERRIDES) W.RETRAIN_OVERRIDES["n_jobs"] = lab.THREADS # shared laptop; winners uses -1 try: m, st = W.train_retrained(Xm, ym, Xa, Xa[shot], ys) finally: W.RETRAIN_OVERRIDES.clear() W.RETRAIN_OVERRIDES.update(saved) return W.predict_retrained(m, st, Xa) def member_metric_maha(enc, shot, ys): M = STATE.metric() return np.mean([M.head_maha(pp["Ea"], shot, ys, pp["mu"], pp["Sm"]) for pp in STATE.metric_prep(enc)], 0) def member_tab_ft(shot, ys): T, embs, lam = STATE.tab() return np.mean([T.head_ft(e, e["Za"][shot], ys, lam) for e in embs], 0) def ensemble(P_zs, P_shot, n): """ensemble.py's rule (log-prob mean, w0 = K/(n+K)).""" logm = lambda Ps: np.mean([np.log(np.clip(P, 1e-6, 1)) for P in Ps], 0) w0 = K_ENS / (n + K_ENS) return lab.softmax(w0 * logm([P_zs]) + (1 - w0) * logm(P_shot)) # --------------------------------------------------------------------------- # class FinalAvgModel: def __init__(self, proba, keep, spatial, members, proba_ensemble): self.proba = proba # (25992, 4), before smoothing / the rule self.keep = keep # non-shot rows, in order self.spatial = spatial self.members = members # name -> (25992, 4) self.proba_ensemble = proba_ensemble # before label propagation def predict(self) -> np.ndarray: """Class labels 1..4 for every row in self.keep (batch-level decision).""" d = STATE.data() P = self.proba[self.keep] if self.spatial: P = lab.smooth(P, d["pa"], keep=self.keep) return W.decide(P, d["prior"]) def model_create(shot_idx, shot_labels, n=None, spatial=True, verbose=False, sort_shots=False) -> FinalAvgModel: """Build FINAL-AVG (spatial=True) or its coordinate-free fallback (spatial=False) from the labelled shots. shot_idx: row indices into the Amsterdam feature table; shot_labels: their classes 1..4; n: shots per class (default len(shot_idx) / 4), used only for the zero-shot weight w0 = 10 / (n + 10). SHOT ORDER: lgbm_upw bags rows by position (subsample=0.8, subsample_freq=1), so the same shots in a different order give a different LightGBM fit (seen: max |dp| 0.33 on lgbm_upw, ~0.02 after the ensemble + LP, macro F1 -0.0006 on n=25 rep=2) - noise of the size of a LightGBM seed change, not float rounding. sort_shots=True sorts (shot, label) by row index first, which makes the method invariant to input order; the default False keeps the caller's order so the stored per-draw files (drawn in lab.shots_for order) are reproduced exactly.""" shot = np.asarray(shot_idx, dtype=np.int64) ys = np.asarray(shot_labels).astype(int) assert len(shot) == len(ys) and set(np.unique(ys)) <= {1, 2, 3, 4} assert len(np.unique(shot)) == len(shot), "duplicate shot indices" if sort_shots: o = np.argsort(shot, kind="stable") shot, ys = shot[o], ys[o] if n is None: n = len(shot) / 4.0 t0 = time.time() def tick(name): if verbose: print(f" [model_create] {name:<18s} {time.time() - t0:6.1f}s", flush=True) members = {"zs_xgb_dann": STATE.zero_shot()} tick("zero-shot") members["lgbm_upw"] = _r32(member_lgbm_upw(shot, ys)); tick("lgbm_upw") for enc in METRIC_MEMBERS: members[f"met_{enc}_maha"] = _r32(member_metric_maha(enc, shot, ys)); tick(f"met_{enc}_maha") members["tab_dann_rz_ft"] = _r32(member_tab_ft(shot, ys)); tick("tab_dann_rz_ft") shot_members = [members[k] for k in ("lgbm_upw", "met_protoda_maha", "met_cos_maha", "tab_dann_rz_ft")] P_ens = _r32(ensemble(members["zs_xgb_dann"], shot_members, n)) proba = P_ens if spatial: TR, g = STATE.graph() proba = TR.r_lp(P_ens, shot, ys, g); tick("label propagation") keep = np.setdiff1d(np.arange(len(proba)), shot) return FinalAvgModel(proba, keep, spatial, members, P_ens) # --------------------------------------------------------------------------- # # Score-only helper (the only code that reads non-shot Amsterdam labels) # --------------------------------------------------------------------------- # def score_only(d, model) -> float: """Macro F1 of model.predict() on its non-shot rows, as lab.score_method computes rule (spatial=False) / rule_smooth (spatial=True).""" return W.macro_f1(d["_ya"][model.keep], model.predict()) def main(): import argparse import json ap = argparse.ArgumentParser() ap.add_argument("--n", type=int, required=True) ap.add_argument("--rep", type=int, required=True) ap.add_argument("--no-spatial", action="store_true") a = ap.parse_args() spatial = not a.no_spatial d = STATE.data() shot = lab.shots_for(d, a.n, a.rep) ys = lab.shot_labels(d, shot) t = time.time() model = model_create(shot, ys, n=a.n, spatial=spatial, verbose=True) labels = model.predict() dt = time.time() - t method, col = ("ens_e4__lp", "rule_smooth") if spatial else ("ens_e4", "rule") f1 = score_only(d, model) print(f"n={a.n} rep={a.rep} spatial={spatial}: {len(labels)} rows labelled in {dt:.1f}s " f"(incl. one-off loading)") print(f"macro F1 ({col}) = {f1:.6f}") sp = lab.SCORES / f"{method}.json" if sp.exists(): rows = [r for r in json.loads(sp.read_text())["draws"] if r["n"] == a.n and r["rep"] == a.rep] if rows: print(f"stored {method} {col} = {rows[0][col]:.6f} diff {f1 - rows[0][col]:+.2e}") pp = lab.pred_path(method, a.n, a.rep) # comparison only if pp.exists(): S = np.load(pp).astype(np.float64) print(f"max |proba - stored {method}| = {np.abs(model.proba - S).max():.3e}") for k in model.members: mp = lab.pred_path(k, a.n, a.rep) if k != "zs_xgb_dann" else lab.pred_path(k, 0, 0) if mp.exists(): print(f" member {k:<18s} max |diff| = {np.abs(model.members[k] - np.load(mp)).max():.3e}") if spatial: ep = lab.pred_path("ens_e4", a.n, a.rep) if ep.exists(): print(f" ensemble (pre-LP) max |diff| = {np.abs(model.proba_ensemble - np.load(ep)).max():.3e}") else: print(f"no stored {pp} to compare with") if __name__ == "__main__": main()