Download pyramidformer.py from SallySims/pyramidformer-base: direct link, hf CLI and curl.
- Browser
- Download file 28.1 kB
-
https://huggingface.co/SallySims/pyramidformer-base/resolve/main/pyramidformer.py
- Command line
-
hf download hf://SallySims/pyramidformer-base/pyramidformer.py
-
curl -L -o pyramidformer.py https://huggingface.co/SallySims/pyramidformer-base/resolve/main/pyramidformer.py
28.1 kB
| """ | |
| pyramidformer.py | |
| ================ | |
| PyramidFormer: a small transformer encoder over the 101 single-year age tokens of a | |
| population pyramid (UN WPP), with task heads. | |
| A pyramid is a (2, 101) array [male, female] x ages 0..100 (100 = 100+) of population | |
| shares (sums to 1). Shares are log-transformed and standardised (Normaliser) before the model. | |
| Tasks (PyramidFormer(task=...)): | |
| base masked-age modelling (self-supervised pretraining): reconstruct hidden ages | |
| similarity embedding of the pyramid shape (year is NOT an input): contrastive + auxiliary stats | |
| regression + a distance-preserving term | |
| heaping per-age logit: is this age inflated by digit preference (0/5 heaping)? | |
| forecast predict the pyramid h years ahead (residual on the input pyramid), h = 1..25 | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import os | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| N_AGES = 101 | |
| HEAP_AGES = list(range(5, 100, 5)) | |
| MAX_H = 25 | |
| # ---------------------------------------------------------------------------- | |
| # Data (UN WPP 2024 Excel files, single age, wide format) | |
| # ---------------------------------------------------------------------------- | |
| def _read_wpp_xlsx(path, sheet): | |
| """Read one UN WPP 'single age' Excel sheet into a DataFrame: iso3, location, year, 0..100.""" | |
| xl = pd.ExcelFile(path) | |
| names = xl.sheet_names | |
| hit = [n for n in names if n.lower().strip() == sheet.lower()] or \ | |
| [n for n in names if sheet.lower() in n.lower()] | |
| if not hit: | |
| raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}") | |
| raw = xl.parse(hit[0], header=None) | |
| hdr = None | |
| for i in range(min(len(raw), 60)): | |
| cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()] | |
| if "year" in cells and any(c.startswith("iso3") for c in cells): | |
| hdr = i | |
| break | |
| if hdr is None: | |
| raise ValueError(f"Could not find the header row in sheet '{hit[0]}' of {path}.") | |
| cols = [str(v).strip() for v in raw.iloc[hdr].tolist()] | |
| df = raw.iloc[hdr + 1:].copy() | |
| df.columns = cols | |
| low = {c.lower(): c for c in cols} | |
| c_iso = next(c for c in cols if c.lower().startswith("iso3")) | |
| c_year = low["year"] | |
| c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso) | |
| c_typ = low.get("type") | |
| age_cols = {} | |
| for c in cols: | |
| t = c.replace(".0", "").strip() | |
| if t.isdigit() and 0 <= int(t) <= 100: | |
| age_cols[int(t)] = c | |
| elif t.startswith("100"): | |
| age_cols[100] = c | |
| missing = [a for a in range(N_AGES) if a not in age_cols] | |
| if missing: | |
| raise ValueError(f"Age columns {missing[:5]}... not found in '{hit[0]}'; is this the SINGLE AGE file?") | |
| if c_typ is not None: | |
| df = df[df[c_typ].astype(str).str.startswith("Country")] | |
| out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str), | |
| "year": pd.to_numeric(df[c_year], errors="coerce")}) | |
| ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy() | |
| for a in range(N_AGES)}, index=out.index) | |
| out = pd.concat([out, ages_df], axis=1) | |
| out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna() | |
| out["year"] = out["year"].astype(int) | |
| return out | |
| def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023): | |
| """Returns (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands).""" | |
| m = _read_wpp_xlsx(male_path, sheet) | |
| f = _read_wpp_xlsx(female_path, sheet) | |
| ages = list(range(N_AGES)) | |
| m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) | |
| f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) | |
| common = m.index.intersection(f.index).sort_values() | |
| if len(common) == 0: | |
| raise ValueError("No overlapping (country, year) rows between the male and female files.") | |
| counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1) | |
| meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(), | |
| "year": [y for _, y in common]}) | |
| return meta, counts | |
| def _norm_shares(a): | |
| a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES) | |
| return a / a.sum(axis=(1, 2), keepdims=True) | |
| def counts_to_shares(counts): | |
| counts = np.asarray(counts, dtype=np.float64) | |
| return counts / counts.sum(axis=(-2, -1), keepdims=True) | |
| def split_by_country(meta, val_frac=0.10, test_frac=0.10, seed=0): | |
| """Country-level split: no country appears in more than one split.""" | |
| rng = np.random.default_rng(seed) | |
| iso = np.array(sorted(meta["iso3"].unique())) | |
| rng.shuffle(iso) | |
| n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac) | |
| test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val] | |
| is_test = meta["iso3"].isin(test_iso).to_numpy() | |
| is_val = meta["iso3"].isin(val_iso).to_numpy() | |
| return ~(is_test | is_val), is_val, is_test | |
| def build_grid(meta, shares): | |
| """Arrange pyramids as (country, year, 2, 101). Returns (iso list, years array, grid of shares).""" | |
| iso = sorted(meta["iso3"].unique()) | |
| yrs = np.sort(meta["year"].unique()) | |
| assert len(yrs) == yrs[-1] - yrs[0] + 1, "years are not contiguous" | |
| ci = pd.Categorical(meta["iso3"], categories=iso).codes | |
| yi = (meta["year"].to_numpy() - yrs[0]).astype(int) | |
| grid = np.full((len(iso), len(yrs), 2, N_AGES), np.nan) | |
| grid[ci, yi] = _norm_shares(shares) | |
| return iso, yrs, grid | |
| def summary_stats(shares): | |
| """(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females).""" | |
| s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES) | |
| tot = s.sum(axis=1) | |
| tot = tot / tot.sum(axis=-1, keepdims=True) | |
| cdf = np.cumsum(tot, axis=-1) | |
| idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1) | |
| rows = np.arange(len(s)) | |
| prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0) | |
| frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12) | |
| med = idx + np.clip(frac, 0.0, 1.0) | |
| u15 = tot[:, :15].sum(-1) | |
| p65 = tot[:, 65:].sum(-1) | |
| sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12) | |
| return np.stack([med, u15, p65, sr], axis=-1) | |
| def stat_targets(shares): | |
| """Standardised (median age, share <15, share 65+) -> (n, 3) float32; auxiliary targets for the embedding.""" | |
| st = summary_stats(np.asarray(shares).reshape(-1, 2, N_AGES)) | |
| return np.stack([(st[:, 0] - 30) / 15, (st[:, 1] - 0.30) / 0.15, (st[:, 2] - 0.10) / 0.10], -1).astype(np.float32) | |
| def year_feat(years): | |
| return ((np.asarray(years, dtype=np.float32) - 1985.0) / 50.0).astype(np.float32) | |
| class Normaliser: | |
| """log(share * 2 * N_AGES + eps), standardised with global mean / std.""" | |
| def __init__(self, mu=0.0, sigma=1.0, eps=1e-5): | |
| self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps) | |
| def _log(self, shares): | |
| return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps) | |
| def fit(self, shares): | |
| z = self._log(shares) | |
| self.mu, self.sigma = float(z.mean()), float(z.std()) | |
| return self | |
| def encode(self, shares): | |
| return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32) | |
| def decode(self, x): | |
| x = np.asarray(x, dtype=np.float64) | |
| s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES) | |
| return s / s.sum(axis=(-2, -1), keepdims=True) | |
| # ---------------------------------------------------------------------------- | |
| # Synthetic age heaping (with per-age labels) and classical baselines | |
| # ---------------------------------------------------------------------------- | |
| def heap_pyramids(shares, lam, rng, p_active=0.6, noise_sd=0.03, thr=0.05): | |
| """Digit preference at multiples of 5. Each pyramid gets its own random subset of 'active' | |
| heaps with random strengths, so the pattern is not identical across pyramids. | |
| A fraction lam*strength of people within +-2 years of a multiple of 5 report that multiple. | |
| Returns (observed shares, labels (n, 101)): label = 1 where the age's count is inflated by > thr.""" | |
| truth = _norm_shares(shares) | |
| out = truth.copy() | |
| n = len(out) | |
| lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (n,)) | |
| act = (rng.random((n, len(HEAP_AGES))) < p_active) * rng.uniform(0.4, 1.6, (n, len(HEAP_AGES))) | |
| for j, h in enumerate(HEAP_AGES): | |
| boost = 1.6 if h % 10 == 0 else 1.0 | |
| for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)): | |
| frac = np.clip(lam * act[:, j] * boost * w, 0, 0.9)[:, None] | |
| moved = out[:, :, h + d] * frac | |
| out[:, :, h + d] -= moved | |
| out[:, :, h] += moved | |
| t, o = truth.sum(1), out.sum(1) | |
| label = ((o / np.maximum(t, 1e-7) - 1.0) > thr) & (t > 1e-7) | |
| out = out * np.exp(rng.normal(0.0, noise_sd, out.shape)) | |
| out = out / out.sum(axis=(1, 2), keepdims=True) | |
| return out, label.astype(np.float32) | |
| def make_heaping_set(shares, years, rng, copies=4, lam_range=(0.05, 0.35), p_clean=0.2): | |
| """Returns observed shares, per-age labels, years, lam (0 = clean pyramid).""" | |
| base = np.repeat(_norm_shares(shares), copies, axis=0) | |
| yrs = np.repeat(np.asarray(years), copies) | |
| lam = rng.uniform(*lam_range, len(base)) | |
| lam[rng.random(len(base)) < p_clean] = 0.0 | |
| obs, y = heap_pyramids(base, lam, rng) | |
| return obs, y, yrs, lam | |
| def whipple(shares): | |
| """Whipple index (both sexes), ages 23-62. ~100 = no preference.""" | |
| tot = _norm_shares(shares).sum(axis=1) | |
| ages = np.arange(23, 63) | |
| return 500.0 * tot[:, ages[ages % 5 == 0]].sum(-1) / tot[:, 23:63].sum(-1) | |
| def cohort_shift(shares, h): | |
| """Classical forecast baseline: age everyone by h years (no mortality), new cohorts = current age-0 size.""" | |
| s = _norm_shares(shares) | |
| out = np.zeros_like(s) | |
| for a in range(N_AGES): | |
| out[..., min(a + h, N_AGES - 1)] += s[..., a] | |
| out[..., :h] += s[..., :1] | |
| return out / out.sum(axis=(-2, -1), keepdims=True) | |
| def mae_pm(a, b): | |
| """Mean absolute error of shares, in per mille of total population.""" | |
| return float(np.abs(np.asarray(a) - np.asarray(b)).mean() * 1e3) | |
| # ---------------------------------------------------------------------------- | |
| # Model | |
| # ---------------------------------------------------------------------------- | |
| class PyramidFormer(nn.Module): | |
| def __init__(self, task="base", d=128, depth=4, heads=4, ff=512, dropout=0.1, max_h=MAX_H, proj_dim=64): | |
| super().__init__() | |
| self.cfg = dict(task=task, d=d, depth=depth, heads=heads, ff=ff, dropout=dropout, | |
| max_h=max_h, proj_dim=proj_dim) | |
| self.task = task | |
| self.inp = nn.Linear(2, d) # (male, female) z-values per age | |
| self.age = nn.Parameter(torch.randn(1, N_AGES, d) * 0.02) # learned age embedding | |
| self.year = nn.Sequential(nn.Linear(1, d), nn.SiLU(), nn.Linear(d, d)) | |
| self.mask_tok = nn.Parameter(torch.zeros(1, 1, d)) | |
| layer = nn.TransformerEncoderLayer(d_model=d, nhead=heads, dim_feedforward=ff, dropout=dropout, | |
| activation="gelu", batch_first=True, norm_first=True) | |
| self.enc = nn.TransformerEncoder(layer, num_layers=depth, enable_nested_tensor=False) | |
| self.norm = nn.LayerNorm(d) | |
| if task == "base": | |
| self.recon = nn.Linear(d, 2) | |
| elif task == "similarity": | |
| self.proj = nn.Sequential(nn.Linear(d, d), nn.GELU(), nn.Linear(d, proj_dim)) | |
| self.aux = nn.Linear(proj_dim, 3) # auxiliary: standardised summary stats | |
| elif task == "heaping": | |
| self.tok = nn.Sequential(nn.Linear(d, d // 2), nn.GELU(), nn.Linear(d // 2, 1)) | |
| elif task == "forecast": | |
| self.hemb = nn.Embedding(max_h + 1, d) | |
| self.delta = nn.Linear(d, 2) | |
| nn.init.zeros_(self.delta.weight) # start as persistence | |
| nn.init.zeros_(self.delta.bias) | |
| else: | |
| raise ValueError(f"unknown task {task}") | |
| def encode(self, x, year, mask=None, h=None): | |
| t = self.inp(x.transpose(1, 2)) # (B, 101, d) | |
| if mask is not None: | |
| t = torch.where(mask[..., None], self.mask_tok.expand_as(t), t) | |
| cond = self.year(year[:, None]) # (B, d) | |
| if h is not None: | |
| cond = cond + self.hemb(h) | |
| t = t + self.age + cond[:, None, :] | |
| return self.norm(self.enc(t)) | |
| def aux_stats(self, emb): | |
| return self.aux(emb) | |
| def forward(self, x, year, mask=None, h=None): | |
| """x: (B, 2, 101) z-encoded; year: (B,) from year_feat; mask: (B, 101) bool; h: (B,) long.""" | |
| if self.task == "similarity": | |
| year = torch.zeros_like(year) # embedding must not depend on year | |
| z = self.encode(x, year, mask, h) | |
| if self.task == "base": | |
| return self.recon(z).transpose(1, 2) # (B, 2, 101) | |
| if self.task == "similarity": | |
| return F.normalize(self.proj(z.mean(1)), dim=-1) # (B, proj_dim) | |
| if self.task == "heaping": | |
| return self.tok(z).squeeze(-1) # (B, 101) logits | |
| return x + self.delta(z).transpose(1, 2) # forecast (B, 2, 101) | |
| def init_from_base(model, base): | |
| """Copy the pretrained backbone into a task model (task heads stay freshly initialised).""" | |
| sd = {k: v for k, v in base.state_dict().items() if not k.startswith("recon.")} | |
| res = model.load_state_dict(sd, strict=False) | |
| print("backbone loaded; new (head) params:", sorted({k.split('.')[0] for k in res.missing_keys}), | |
| "| unexpected:", res.unexpected_keys) | |
| return model | |
| # ---------------------------------------------------------------------------- | |
| # Training | |
| # ---------------------------------------------------------------------------- | |
| def fit(model, loss_fn, steps, lr=5e-4, wd=1e-2, log_every=500, val_fn=None, device=None, seed=0): | |
| device = device or ("cuda" if torch.cuda.is_available() else "cpu") | |
| torch.manual_seed(seed) | |
| model.to(device).train() | |
| opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd) | |
| history, run = [], [] | |
| for step in range(steps): | |
| lr_now = lr * min(1.0, (step + 1) / 300) * 0.5 * (1 + math.cos(math.pi * step / steps)) | |
| for g in opt.param_groups: | |
| g["lr"] = lr_now | |
| loss = loss_fn(model, device) | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| opt.step() | |
| run.append(loss.item()) | |
| if (step + 1) % log_every == 0 or step + 1 == steps: | |
| rec = {"step": step + 1, "train_loss": float(np.mean(run))} | |
| run = [] | |
| if val_fn is not None: | |
| model.eval() | |
| with torch.no_grad(): | |
| rec["val_loss"] = float(val_fn(model, device)) | |
| model.train() | |
| history.append(rec) | |
| print(rec) | |
| model.eval() | |
| return history | |
| def make_mask(B, device, p=0.3, gen=None): | |
| """Half the batch: random ages hidden; other half: one contiguous span of 5-20 ages hidden.""" | |
| rnd = torch.rand(B, N_AGES, device=device, generator=gen) < p | |
| use_span = torch.rand(B, device=device, generator=gen) < 0.5 | |
| L = torch.randint(5, 21, (B, 1), device=device, generator=gen) | |
| st = (torch.rand(B, 1, device=device, generator=gen) * (N_AGES - L)).long() | |
| ar = torch.arange(N_AGES, device=device)[None] | |
| span = (ar >= st) & (ar < st + L) | |
| return torch.where(use_span[:, None], span, rnd) | |
| def mam_loss_fn(X, Y, bs=256, p=0.3): | |
| def f(model, device): | |
| idx = torch.randint(0, X.shape[0], (bs,), device=device) | |
| x, yr = X[idx], Y[idx] | |
| m = make_mask(bs, device, p) | |
| pred = model(x, yr, mask=m) | |
| return ((pred - x) ** 2).mean(1)[m].mean() | |
| return f | |
| def mam_val_fn(X, Y, bs=512): | |
| def f(model, device): | |
| g = torch.Generator(device=device).manual_seed(123) | |
| tot, n = 0.0, 0 | |
| for i in range(0, X.shape[0], bs): | |
| x, yr = X[i:i + bs], Y[i:i + bs] | |
| m = make_mask(len(x), device, 0.3, gen=g) | |
| pred = model(x, yr, mask=m) | |
| tot += ((pred - x) ** 2).mean(1)[m].sum().item() | |
| n += int(m.sum().item()) | |
| return tot / max(n, 1) | |
| return f | |
| def sim_loss_fn(Z, S, c_idx, bs=256, tau=0.1, noise=0.02, aux_w=1.0, geo_w=2.0): | |
| """Z: (C, Y, 2, 101) z-encoded grid; S: (C, Y, 3) standardised summary stats; c_idx: training countries. | |
| 1. InfoNCE: positive = the same country 1-2 years earlier/later; same-country pairs are never negatives. | |
| 2. aux_w: the embedding must predict median age, share <15, share 65+ (linear head). | |
| 3. geo_w: cosine similarity of two pyramids in the batch should follow their L1 distance | |
| (counteracts InfoNCE pushing apart different countries that have the same shape).""" | |
| Y = Z.shape[1] | |
| offs = torch.tensor([-2, -1, 1, 2]) | |
| def f(model, device): | |
| o = offs.to(device) | |
| c = c_idx[torch.randint(len(c_idx), (bs,), device=device)] | |
| t = torch.randint(0, Y, (bs,), device=device) | |
| t2 = (t + o[torch.randint(4, (bs,), device=device)]).clamp(0, Y - 1) | |
| a0, b0 = Z[c, t], Z[c, t2] | |
| a = a0 + noise * torch.randn_like(a0) | |
| b = b0 + noise * torch.randn_like(b0) | |
| zero = torch.zeros(bs, device=device) | |
| ea, eb = model(a, zero), model(b, zero) | |
| logits = ea @ eb.T / tau | |
| same = (c[:, None] == c[None, :]) & ~torch.eye(bs, dtype=torch.bool, device=device) | |
| logits = logits.masked_fill(same, -1e9) | |
| lab = torch.arange(bs, device=device) | |
| loss = 0.5 * (F.cross_entropy(logits, lab) + F.cross_entropy(logits.T, lab)) | |
| if aux_w > 0: | |
| loss = loss + aux_w * 0.5 * (F.mse_loss(model.aux_stats(ea), S[c, t]) + | |
| F.mse_loss(model.aux_stats(eb), S[c, t2])) | |
| if geo_w > 0: | |
| with torch.no_grad(): | |
| D = torch.cdist(a0.flatten(1), a0.flatten(1), p=1) / a0[0].numel() | |
| T = torch.exp(-D / D.median().clamp(min=1e-6)) | |
| loss = loss + geo_w * F.mse_loss(ea @ ea.T, T) | |
| return loss | |
| return f | |
| def heap_loss_fn(X, Yl, yrs, bs=256, pos_weight=3.0): | |
| def f(model, device): | |
| idx = torch.randint(0, X.shape[0], (bs,), device=device) | |
| logits = model(X[idx], yrs[idx]) | |
| pw = torch.tensor(pos_weight, device=device) | |
| return F.binary_cross_entropy_with_logits(logits, Yl[idx], pos_weight=pw) | |
| return f | |
| def heap_val_fn(X, Yl, yrs, bs=1024, pos_weight=3.0): | |
| def f(model, device): | |
| pw = torch.tensor(pos_weight, device=device) | |
| tot, n = 0.0, 0 | |
| for i in range(0, X.shape[0], bs): | |
| lg = model(X[i:i + bs], yrs[i:i + bs]) | |
| tot += F.binary_cross_entropy_with_logits(lg, Yl[i:i + bs], pos_weight=pw, reduction="sum").item() | |
| n += lg.numel() | |
| return tot / n | |
| return f | |
| def forecast_loss_fn(Z, c_idx, year0, max_h=MAX_H, bs=256): | |
| Y = Z.shape[1] | |
| def f(model, device): | |
| c = c_idx[torch.randint(len(c_idx), (bs,), device=device)] | |
| h = torch.randint(1, max_h + 1, (bs,), device=device) | |
| t0 = (torch.rand(bs, device=device) * (Y - h)).long() # guarantees t0 + h <= Y - 1 | |
| yr = (year0 + t0.float() - 1985.0) / 50.0 | |
| return F.mse_loss(model(Z[c, t0], yr, h=h), Z[c, t0 + h]) | |
| return f | |
| def forecast_val_fn(Z, c_idx, year0, hs=(1, 5, 10, 20, 25)): | |
| Y = Z.shape[1] | |
| def f(model, device): | |
| Zc = Z[c_idx] | |
| losses = [] | |
| for h in hs: | |
| x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES) | |
| x1 = Zc[:, h:].reshape(-1, 2, N_AGES) | |
| yr = ((year0 + torch.arange(Y - h, device=device).float() - 1985.0) / 50.0).repeat(len(c_idx)) | |
| hh = torch.full((len(x0),), h, dtype=torch.long, device=device) | |
| losses.append(F.mse_loss(model(x0, yr, h=hh), x1).item()) | |
| return float(np.mean(losses)) | |
| return f | |
| # ---------------------------------------------------------------------------- | |
| # Inference helpers and evaluation | |
| # ---------------------------------------------------------------------------- | |
| def _dev(model): | |
| return next(model.parameters()).device | |
| def embed(model, Z, bs=1024): | |
| """Z: z-encoded pyramids (n, 2, 101) -> L2-normalised embeddings (n, proj_dim).""" | |
| model.eval() | |
| dev = _dev(model) | |
| out = [] | |
| for i in range(0, len(Z), bs): | |
| x = torch.as_tensor(Z[i:i + bs]).to(dev) | |
| out.append(model(x, torch.zeros(len(x), device=dev)).cpu()) | |
| return torch.cat(out).numpy() | |
| def most_similar(model, norm, query_shares, E_db, meta, k=5, exclude_iso=None, per_country=True): | |
| """Nearest pyramids in a database of embeddings E_db (rows aligned with meta). | |
| per_country=True keeps only the best-matching year of each country, so results are k different countries.""" | |
| q = embed(model, norm.encode(_norm_shares(query_shares))) | |
| S = q @ E_db.T | |
| iso = meta["iso3"].to_numpy() | |
| if exclude_iso is not None: | |
| S[:, iso == exclude_iso] = -1.0 | |
| out = [] | |
| for j in range(len(q)): | |
| keep, seen = [], set() | |
| for i in np.argsort(-S[j]): | |
| if per_country and iso[i] in seen: | |
| continue | |
| seen.add(iso[i]) | |
| keep.append(int(i)) | |
| if len(keep) == k: | |
| break | |
| out.append(meta.iloc[keep][["iso3", "location", "year"]].assign(cosine=S[j, keep]).reset_index(drop=True)) | |
| return out | |
| def _knn_r2(Ftr, Fte, ytr, yte, k=5, cosine=True): | |
| Ft, Fe = torch.as_tensor(Ftr), torch.as_tensor(Fte) | |
| if cosine: | |
| idx = (Fe @ Ft.T).topk(k, dim=1).indices | |
| else: | |
| idx = torch.cdist(Fe, Ft).topk(k, dim=1, largest=False).indices | |
| pred = torch.as_tensor(ytr)[idx].mean(1).numpy() | |
| return float(1 - ((yte - pred) ** 2).sum() / ((yte - yte.mean()) ** 2).sum()) | |
| def evaluate_similarity(model, Ztr, sh_tr, Zte, sh_te, k=5, n_pairs=20000, seed=0): | |
| """(1) kNN regression of summary stats from the embedding vs from the raw pyramid, on held-out | |
| countries (database = training pyramids); (2) Spearman between embedding cosine and -L1 distance.""" | |
| from scipy.stats import spearmanr | |
| etr, ete = embed(model, Ztr), embed(model, Zte) | |
| s_tr, s_te = summary_stats(sh_tr), summary_stats(sh_te) | |
| names = ["median_age", "share_under_15", "share_65_plus"] | |
| res = {"knn_r2_embedding": {}, "knn_r2_raw_pyramid": {}} | |
| raw_tr, raw_te = Ztr.reshape(len(Ztr), -1), Zte.reshape(len(Zte), -1) | |
| for j, nme in enumerate(names): | |
| res["knn_r2_embedding"][nme] = _knn_r2(etr, ete, s_tr[:, j], s_te[:, j], k, True) | |
| res["knn_r2_raw_pyramid"][nme] = _knn_r2(raw_tr, raw_te, s_tr[:, j], s_te[:, j], k, False) | |
| rng = np.random.default_rng(seed) | |
| i, j = rng.integers(0, len(Zte), n_pairs), rng.integers(0, len(Zte), n_pairs) | |
| cos = (ete[i] * ete[j]).sum(1) | |
| l1 = np.abs(_norm_shares(sh_te)[i] - _norm_shares(sh_te)[j]).sum((1, 2)) | |
| res["spearman_cosine_vs_neg_L1"] = float(spearmanr(cos, -l1).correlation) | |
| return res | |
| def predict_heaping(model, Z, yrs, bs=1024): | |
| model.eval() | |
| dev = _dev(model) | |
| out = [] | |
| for i in range(0, len(Z), bs): | |
| x = torch.as_tensor(Z[i:i + bs]).to(dev) | |
| y = torch.as_tensor(year_feat(yrs[i:i + bs])).to(dev) | |
| out.append(torch.sigmoid(model(x, y)).cpu().numpy()) | |
| return np.concatenate(out) | |
| def _prf(pred, y): | |
| tp = float(((pred == 1) & (y == 1)).sum()) | |
| fp = float(((pred == 1) & (y == 0)).sum()) | |
| fn = float(((pred == 0) & (y == 1)).sum()) | |
| p, r = tp / max(tp + fp, 1), tp / max(tp + fn, 1) | |
| return {"precision": p, "recall": r, "f1": 2 * p * r / max(p + r, 1e-12)} | |
| def evaluate_heaping(model, norm, obs, y, yrs, lam): | |
| """Per-age detection on held-out countries vs a digit rule that flags every age ending in 0/5.""" | |
| from sklearn.metrics import roc_auc_score, average_precision_score | |
| prob = predict_heaping(model, norm.encode(obs), yrs) | |
| rule = np.zeros(N_AGES) | |
| rule[HEAP_AGES] = 1.0 | |
| rule = np.broadcast_to(rule, y.shape) | |
| clean = lam == 0 | |
| pred = (prob >= 0.5).astype(int) | |
| res = {"n_pyramids": int(len(y)), "positive_rate": float(y.mean()), | |
| "pyramidformer": {"auroc": float(roc_auc_score(y.ravel(), prob.ravel())), | |
| "avg_precision": float(average_precision_score(y.ravel(), prob.ravel())), | |
| **_prf(pred, y), "false_positive_rate_on_clean_pyramids": float(pred[clean].mean())}, | |
| "digit_rule_0_5": {**_prf(rule.astype(int), y), | |
| "false_positive_rate_on_clean_pyramids": float(rule[clean].mean())}} | |
| return res | |
| def predict_forecast(model, norm, x0, years0, h, bs=2048): | |
| model.eval() | |
| dev = _dev(model) | |
| z0 = norm.encode(_norm_shares(x0)) | |
| yf = year_feat(np.broadcast_to(np.asarray(years0), (len(z0),))) | |
| out = [] | |
| for i in range(0, len(z0), bs): | |
| z = torch.as_tensor(z0[i:i + bs]).to(dev) | |
| y = torch.as_tensor(yf[i:i + bs]).to(dev) | |
| hh = torch.full((len(z),), int(h), dtype=torch.long, device=dev) | |
| out.append(model(z, y, h=hh).cpu().numpy()) | |
| return norm.decode(np.concatenate(out)) | |
| def eval_pairs(model, norm, x0, x1, years0, h): | |
| x0, x1 = _norm_shares(x0), _norm_shares(x1) | |
| pred = predict_forecast(model, norm, x0, years0, h) | |
| cs = cohort_shift(x0, h) | |
| m1 = summary_stats(x1)[:, 0] | |
| return {"n": int(len(x0)), | |
| "mae_permille": {"pyramidformer": mae_pm(pred, x1), "persistence": mae_pm(x0, x1), | |
| "cohort_shift": mae_pm(cs, x1)}, | |
| "median_age_abs_err_years": {"pyramidformer": float(np.abs(summary_stats(pred)[:, 0] - m1).mean()), | |
| "persistence": float(np.abs(summary_stats(x0)[:, 0] - m1).mean()), | |
| "cohort_shift": float(np.abs(summary_stats(cs)[:, 0] - m1).mean())}} | |
| def evaluate_forecast(model, norm, grid, idx, year0, hs=(1, 5, 10, 20, 25)): | |
| """All (origin year, origin year + h) pairs of the countries in idx (held-out countries).""" | |
| Zc = grid[idx] | |
| Y = Zc.shape[1] | |
| res = {} | |
| for h in hs: | |
| x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES) | |
| x1 = Zc[:, h:].reshape(-1, 2, N_AGES) | |
| yrs0 = np.tile(year0 + np.arange(Y - h), len(idx)) | |
| res[h] = eval_pairs(model, norm, x0, x1, yrs0, h) | |
| return res | |
| # ---------------------------------------------------------------------------- | |
| # Save / load | |
| # ---------------------------------------------------------------------------- | |
| def save_pretrained(path, model, norm, extra=None): | |
| from safetensors.torch import save_file | |
| os.makedirs(path, exist_ok=True) | |
| sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()} | |
| save_file(sd, os.path.join(path, "model.safetensors")) | |
| cfg = {"arch": model.cfg, "n_ages": N_AGES, | |
| "norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}} | |
| with open(os.path.join(path, "config.json"), "w") as f: | |
| json.dump(cfg, f, indent=2) | |
| def load_pretrained(path, device="cpu"): | |
| from safetensors.torch import load_file | |
| with open(os.path.join(path, "config.json")) as f: | |
| cfg = json.load(f) | |
| model = PyramidFormer(**cfg["arch"]) | |
| model.load_state_dict(load_file(os.path.join(path, "model.safetensors"))) | |
| model.to(device).eval() | |
| return model, Normaliser(**cfg["norm"]) | |