""" pyramidformer.py ================ PyramidFormer: a small transformer encoder over the 101 single-year age tokens of a population pyramid (UN WPP), with task heads. A pyramid is a (2, 101) array [male, female] x ages 0..100 (100 = 100+) of population shares (sums to 1). Shares are log-transformed and standardised (Normaliser) before the model. Tasks (PyramidFormer(task=...)): base masked-age modelling (self-supervised pretraining): reconstruct hidden ages similarity embedding of the pyramid shape (year is NOT an input): contrastive + auxiliary stats regression + a distance-preserving term heaping per-age logit: is this age inflated by digit preference (0/5 heaping)? forecast predict the pyramid h years ahead (residual on the input pyramid), h = 1..25 """ from __future__ import annotations import json import math import os import numpy as np import pandas as pd import torch import torch.nn as nn import torch.nn.functional as F N_AGES = 101 HEAP_AGES = list(range(5, 100, 5)) MAX_H = 25 # ---------------------------------------------------------------------------- # Data (UN WPP 2024 Excel files, single age, wide format) # ---------------------------------------------------------------------------- def _read_wpp_xlsx(path, sheet): """Read one UN WPP 'single age' Excel sheet into a DataFrame: iso3, location, year, 0..100.""" xl = pd.ExcelFile(path) names = xl.sheet_names hit = [n for n in names if n.lower().strip() == sheet.lower()] or \ [n for n in names if sheet.lower() in n.lower()] if not hit: raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}") raw = xl.parse(hit[0], header=None) hdr = None for i in range(min(len(raw), 60)): cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()] if "year" in cells and any(c.startswith("iso3") for c in cells): hdr = i break if hdr is None: raise ValueError(f"Could not find the header row in sheet '{hit[0]}' of {path}.") cols = [str(v).strip() for v in raw.iloc[hdr].tolist()] df = raw.iloc[hdr + 1:].copy() df.columns = cols low = {c.lower(): c for c in cols} c_iso = next(c for c in cols if c.lower().startswith("iso3")) c_year = low["year"] c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso) c_typ = low.get("type") age_cols = {} for c in cols: t = c.replace(".0", "").strip() if t.isdigit() and 0 <= int(t) <= 100: age_cols[int(t)] = c elif t.startswith("100"): age_cols[100] = c missing = [a for a in range(N_AGES) if a not in age_cols] if missing: raise ValueError(f"Age columns {missing[:5]}... not found in '{hit[0]}'; is this the SINGLE AGE file?") if c_typ is not None: df = df[df[c_typ].astype(str).str.startswith("Country")] out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str), "year": pd.to_numeric(df[c_year], errors="coerce")}) ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy() for a in range(N_AGES)}, index=out.index) out = pd.concat([out, ages_df], axis=1) out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna() out["year"] = out["year"].astype(int) return out def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023): """Returns (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands).""" m = _read_wpp_xlsx(male_path, sheet) f = _read_wpp_xlsx(female_path, sheet) ages = list(range(N_AGES)) m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) common = m.index.intersection(f.index).sort_values() if len(common) == 0: raise ValueError("No overlapping (country, year) rows between the male and female files.") counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1) meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(), "year": [y for _, y in common]}) return meta, counts def _norm_shares(a): a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES) return a / a.sum(axis=(1, 2), keepdims=True) def counts_to_shares(counts): counts = np.asarray(counts, dtype=np.float64) return counts / counts.sum(axis=(-2, -1), keepdims=True) def split_by_country(meta, val_frac=0.10, test_frac=0.10, seed=0): """Country-level split: no country appears in more than one split.""" rng = np.random.default_rng(seed) iso = np.array(sorted(meta["iso3"].unique())) rng.shuffle(iso) n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac) test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val] is_test = meta["iso3"].isin(test_iso).to_numpy() is_val = meta["iso3"].isin(val_iso).to_numpy() return ~(is_test | is_val), is_val, is_test def build_grid(meta, shares): """Arrange pyramids as (country, year, 2, 101). Returns (iso list, years array, grid of shares).""" iso = sorted(meta["iso3"].unique()) yrs = np.sort(meta["year"].unique()) assert len(yrs) == yrs[-1] - yrs[0] + 1, "years are not contiguous" ci = pd.Categorical(meta["iso3"], categories=iso).codes yi = (meta["year"].to_numpy() - yrs[0]).astype(int) grid = np.full((len(iso), len(yrs), 2, N_AGES), np.nan) grid[ci, yi] = _norm_shares(shares) return iso, yrs, grid def summary_stats(shares): """(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females).""" s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES) tot = s.sum(axis=1) tot = tot / tot.sum(axis=-1, keepdims=True) cdf = np.cumsum(tot, axis=-1) idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1) rows = np.arange(len(s)) prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0) frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12) med = idx + np.clip(frac, 0.0, 1.0) u15 = tot[:, :15].sum(-1) p65 = tot[:, 65:].sum(-1) sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12) return np.stack([med, u15, p65, sr], axis=-1) def stat_targets(shares): """Standardised (median age, share <15, share 65+) -> (n, 3) float32; auxiliary targets for the embedding.""" st = summary_stats(np.asarray(shares).reshape(-1, 2, N_AGES)) return np.stack([(st[:, 0] - 30) / 15, (st[:, 1] - 0.30) / 0.15, (st[:, 2] - 0.10) / 0.10], -1).astype(np.float32) def year_feat(years): return ((np.asarray(years, dtype=np.float32) - 1985.0) / 50.0).astype(np.float32) class Normaliser: """log(share * 2 * N_AGES + eps), standardised with global mean / std.""" def __init__(self, mu=0.0, sigma=1.0, eps=1e-5): self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps) def _log(self, shares): return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps) def fit(self, shares): z = self._log(shares) self.mu, self.sigma = float(z.mean()), float(z.std()) return self def encode(self, shares): return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32) def decode(self, x): x = np.asarray(x, dtype=np.float64) s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES) return s / s.sum(axis=(-2, -1), keepdims=True) # ---------------------------------------------------------------------------- # Synthetic age heaping (with per-age labels) and classical baselines # ---------------------------------------------------------------------------- def heap_pyramids(shares, lam, rng, p_active=0.6, noise_sd=0.03, thr=0.05): """Digit preference at multiples of 5. Each pyramid gets its own random subset of 'active' heaps with random strengths, so the pattern is not identical across pyramids. A fraction lam*strength of people within +-2 years of a multiple of 5 report that multiple. Returns (observed shares, labels (n, 101)): label = 1 where the age's count is inflated by > thr.""" truth = _norm_shares(shares) out = truth.copy() n = len(out) lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (n,)) act = (rng.random((n, len(HEAP_AGES))) < p_active) * rng.uniform(0.4, 1.6, (n, len(HEAP_AGES))) for j, h in enumerate(HEAP_AGES): boost = 1.6 if h % 10 == 0 else 1.0 for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)): frac = np.clip(lam * act[:, j] * boost * w, 0, 0.9)[:, None] moved = out[:, :, h + d] * frac out[:, :, h + d] -= moved out[:, :, h] += moved t, o = truth.sum(1), out.sum(1) label = ((o / np.maximum(t, 1e-7) - 1.0) > thr) & (t > 1e-7) out = out * np.exp(rng.normal(0.0, noise_sd, out.shape)) out = out / out.sum(axis=(1, 2), keepdims=True) return out, label.astype(np.float32) def make_heaping_set(shares, years, rng, copies=4, lam_range=(0.05, 0.35), p_clean=0.2): """Returns observed shares, per-age labels, years, lam (0 = clean pyramid).""" base = np.repeat(_norm_shares(shares), copies, axis=0) yrs = np.repeat(np.asarray(years), copies) lam = rng.uniform(*lam_range, len(base)) lam[rng.random(len(base)) < p_clean] = 0.0 obs, y = heap_pyramids(base, lam, rng) return obs, y, yrs, lam def whipple(shares): """Whipple index (both sexes), ages 23-62. ~100 = no preference.""" tot = _norm_shares(shares).sum(axis=1) ages = np.arange(23, 63) return 500.0 * tot[:, ages[ages % 5 == 0]].sum(-1) / tot[:, 23:63].sum(-1) def cohort_shift(shares, h): """Classical forecast baseline: age everyone by h years (no mortality), new cohorts = current age-0 size.""" s = _norm_shares(shares) out = np.zeros_like(s) for a in range(N_AGES): out[..., min(a + h, N_AGES - 1)] += s[..., a] out[..., :h] += s[..., :1] return out / out.sum(axis=(-2, -1), keepdims=True) def mae_pm(a, b): """Mean absolute error of shares, in per mille of total population.""" return float(np.abs(np.asarray(a) - np.asarray(b)).mean() * 1e3) # ---------------------------------------------------------------------------- # Model # ---------------------------------------------------------------------------- class PyramidFormer(nn.Module): def __init__(self, task="base", d=128, depth=4, heads=4, ff=512, dropout=0.1, max_h=MAX_H, proj_dim=64): super().__init__() self.cfg = dict(task=task, d=d, depth=depth, heads=heads, ff=ff, dropout=dropout, max_h=max_h, proj_dim=proj_dim) self.task = task self.inp = nn.Linear(2, d) # (male, female) z-values per age self.age = nn.Parameter(torch.randn(1, N_AGES, d) * 0.02) # learned age embedding self.year = nn.Sequential(nn.Linear(1, d), nn.SiLU(), nn.Linear(d, d)) self.mask_tok = nn.Parameter(torch.zeros(1, 1, d)) layer = nn.TransformerEncoderLayer(d_model=d, nhead=heads, dim_feedforward=ff, dropout=dropout, activation="gelu", batch_first=True, norm_first=True) self.enc = nn.TransformerEncoder(layer, num_layers=depth, enable_nested_tensor=False) self.norm = nn.LayerNorm(d) if task == "base": self.recon = nn.Linear(d, 2) elif task == "similarity": self.proj = nn.Sequential(nn.Linear(d, d), nn.GELU(), nn.Linear(d, proj_dim)) self.aux = nn.Linear(proj_dim, 3) # auxiliary: standardised summary stats elif task == "heaping": self.tok = nn.Sequential(nn.Linear(d, d // 2), nn.GELU(), nn.Linear(d // 2, 1)) elif task == "forecast": self.hemb = nn.Embedding(max_h + 1, d) self.delta = nn.Linear(d, 2) nn.init.zeros_(self.delta.weight) # start as persistence nn.init.zeros_(self.delta.bias) else: raise ValueError(f"unknown task {task}") def encode(self, x, year, mask=None, h=None): t = self.inp(x.transpose(1, 2)) # (B, 101, d) if mask is not None: t = torch.where(mask[..., None], self.mask_tok.expand_as(t), t) cond = self.year(year[:, None]) # (B, d) if h is not None: cond = cond + self.hemb(h) t = t + self.age + cond[:, None, :] return self.norm(self.enc(t)) def aux_stats(self, emb): return self.aux(emb) def forward(self, x, year, mask=None, h=None): """x: (B, 2, 101) z-encoded; year: (B,) from year_feat; mask: (B, 101) bool; h: (B,) long.""" if self.task == "similarity": year = torch.zeros_like(year) # embedding must not depend on year z = self.encode(x, year, mask, h) if self.task == "base": return self.recon(z).transpose(1, 2) # (B, 2, 101) if self.task == "similarity": return F.normalize(self.proj(z.mean(1)), dim=-1) # (B, proj_dim) if self.task == "heaping": return self.tok(z).squeeze(-1) # (B, 101) logits return x + self.delta(z).transpose(1, 2) # forecast (B, 2, 101) def init_from_base(model, base): """Copy the pretrained backbone into a task model (task heads stay freshly initialised).""" sd = {k: v for k, v in base.state_dict().items() if not k.startswith("recon.")} res = model.load_state_dict(sd, strict=False) print("backbone loaded; new (head) params:", sorted({k.split('.')[0] for k in res.missing_keys}), "| unexpected:", res.unexpected_keys) return model # ---------------------------------------------------------------------------- # Training # ---------------------------------------------------------------------------- def fit(model, loss_fn, steps, lr=5e-4, wd=1e-2, log_every=500, val_fn=None, device=None, seed=0): device = device or ("cuda" if torch.cuda.is_available() else "cpu") torch.manual_seed(seed) model.to(device).train() opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd) history, run = [], [] for step in range(steps): lr_now = lr * min(1.0, (step + 1) / 300) * 0.5 * (1 + math.cos(math.pi * step / steps)) for g in opt.param_groups: g["lr"] = lr_now loss = loss_fn(model, device) opt.zero_grad(set_to_none=True) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() run.append(loss.item()) if (step + 1) % log_every == 0 or step + 1 == steps: rec = {"step": step + 1, "train_loss": float(np.mean(run))} run = [] if val_fn is not None: model.eval() with torch.no_grad(): rec["val_loss"] = float(val_fn(model, device)) model.train() history.append(rec) print(rec) model.eval() return history def make_mask(B, device, p=0.3, gen=None): """Half the batch: random ages hidden; other half: one contiguous span of 5-20 ages hidden.""" rnd = torch.rand(B, N_AGES, device=device, generator=gen) < p use_span = torch.rand(B, device=device, generator=gen) < 0.5 L = torch.randint(5, 21, (B, 1), device=device, generator=gen) st = (torch.rand(B, 1, device=device, generator=gen) * (N_AGES - L)).long() ar = torch.arange(N_AGES, device=device)[None] span = (ar >= st) & (ar < st + L) return torch.where(use_span[:, None], span, rnd) def mam_loss_fn(X, Y, bs=256, p=0.3): def f(model, device): idx = torch.randint(0, X.shape[0], (bs,), device=device) x, yr = X[idx], Y[idx] m = make_mask(bs, device, p) pred = model(x, yr, mask=m) return ((pred - x) ** 2).mean(1)[m].mean() return f def mam_val_fn(X, Y, bs=512): def f(model, device): g = torch.Generator(device=device).manual_seed(123) tot, n = 0.0, 0 for i in range(0, X.shape[0], bs): x, yr = X[i:i + bs], Y[i:i + bs] m = make_mask(len(x), device, 0.3, gen=g) pred = model(x, yr, mask=m) tot += ((pred - x) ** 2).mean(1)[m].sum().item() n += int(m.sum().item()) return tot / max(n, 1) return f def sim_loss_fn(Z, S, c_idx, bs=256, tau=0.1, noise=0.02, aux_w=1.0, geo_w=2.0): """Z: (C, Y, 2, 101) z-encoded grid; S: (C, Y, 3) standardised summary stats; c_idx: training countries. 1. InfoNCE: positive = the same country 1-2 years earlier/later; same-country pairs are never negatives. 2. aux_w: the embedding must predict median age, share <15, share 65+ (linear head). 3. geo_w: cosine similarity of two pyramids in the batch should follow their L1 distance (counteracts InfoNCE pushing apart different countries that have the same shape).""" Y = Z.shape[1] offs = torch.tensor([-2, -1, 1, 2]) def f(model, device): o = offs.to(device) c = c_idx[torch.randint(len(c_idx), (bs,), device=device)] t = torch.randint(0, Y, (bs,), device=device) t2 = (t + o[torch.randint(4, (bs,), device=device)]).clamp(0, Y - 1) a0, b0 = Z[c, t], Z[c, t2] a = a0 + noise * torch.randn_like(a0) b = b0 + noise * torch.randn_like(b0) zero = torch.zeros(bs, device=device) ea, eb = model(a, zero), model(b, zero) logits = ea @ eb.T / tau same = (c[:, None] == c[None, :]) & ~torch.eye(bs, dtype=torch.bool, device=device) logits = logits.masked_fill(same, -1e9) lab = torch.arange(bs, device=device) loss = 0.5 * (F.cross_entropy(logits, lab) + F.cross_entropy(logits.T, lab)) if aux_w > 0: loss = loss + aux_w * 0.5 * (F.mse_loss(model.aux_stats(ea), S[c, t]) + F.mse_loss(model.aux_stats(eb), S[c, t2])) if geo_w > 0: with torch.no_grad(): D = torch.cdist(a0.flatten(1), a0.flatten(1), p=1) / a0[0].numel() T = torch.exp(-D / D.median().clamp(min=1e-6)) loss = loss + geo_w * F.mse_loss(ea @ ea.T, T) return loss return f def heap_loss_fn(X, Yl, yrs, bs=256, pos_weight=3.0): def f(model, device): idx = torch.randint(0, X.shape[0], (bs,), device=device) logits = model(X[idx], yrs[idx]) pw = torch.tensor(pos_weight, device=device) return F.binary_cross_entropy_with_logits(logits, Yl[idx], pos_weight=pw) return f def heap_val_fn(X, Yl, yrs, bs=1024, pos_weight=3.0): def f(model, device): pw = torch.tensor(pos_weight, device=device) tot, n = 0.0, 0 for i in range(0, X.shape[0], bs): lg = model(X[i:i + bs], yrs[i:i + bs]) tot += F.binary_cross_entropy_with_logits(lg, Yl[i:i + bs], pos_weight=pw, reduction="sum").item() n += lg.numel() return tot / n return f def forecast_loss_fn(Z, c_idx, year0, max_h=MAX_H, bs=256): Y = Z.shape[1] def f(model, device): c = c_idx[torch.randint(len(c_idx), (bs,), device=device)] h = torch.randint(1, max_h + 1, (bs,), device=device) t0 = (torch.rand(bs, device=device) * (Y - h)).long() # guarantees t0 + h <= Y - 1 yr = (year0 + t0.float() - 1985.0) / 50.0 return F.mse_loss(model(Z[c, t0], yr, h=h), Z[c, t0 + h]) return f def forecast_val_fn(Z, c_idx, year0, hs=(1, 5, 10, 20, 25)): Y = Z.shape[1] def f(model, device): Zc = Z[c_idx] losses = [] for h in hs: x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES) x1 = Zc[:, h:].reshape(-1, 2, N_AGES) yr = ((year0 + torch.arange(Y - h, device=device).float() - 1985.0) / 50.0).repeat(len(c_idx)) hh = torch.full((len(x0),), h, dtype=torch.long, device=device) losses.append(F.mse_loss(model(x0, yr, h=hh), x1).item()) return float(np.mean(losses)) return f # ---------------------------------------------------------------------------- # Inference helpers and evaluation # ---------------------------------------------------------------------------- def _dev(model): return next(model.parameters()).device @torch.no_grad() def embed(model, Z, bs=1024): """Z: z-encoded pyramids (n, 2, 101) -> L2-normalised embeddings (n, proj_dim).""" model.eval() dev = _dev(model) out = [] for i in range(0, len(Z), bs): x = torch.as_tensor(Z[i:i + bs]).to(dev) out.append(model(x, torch.zeros(len(x), device=dev)).cpu()) return torch.cat(out).numpy() def most_similar(model, norm, query_shares, E_db, meta, k=5, exclude_iso=None, per_country=True): """Nearest pyramids in a database of embeddings E_db (rows aligned with meta). per_country=True keeps only the best-matching year of each country, so results are k different countries.""" q = embed(model, norm.encode(_norm_shares(query_shares))) S = q @ E_db.T iso = meta["iso3"].to_numpy() if exclude_iso is not None: S[:, iso == exclude_iso] = -1.0 out = [] for j in range(len(q)): keep, seen = [], set() for i in np.argsort(-S[j]): if per_country and iso[i] in seen: continue seen.add(iso[i]) keep.append(int(i)) if len(keep) == k: break out.append(meta.iloc[keep][["iso3", "location", "year"]].assign(cosine=S[j, keep]).reset_index(drop=True)) return out def _knn_r2(Ftr, Fte, ytr, yte, k=5, cosine=True): Ft, Fe = torch.as_tensor(Ftr), torch.as_tensor(Fte) if cosine: idx = (Fe @ Ft.T).topk(k, dim=1).indices else: idx = torch.cdist(Fe, Ft).topk(k, dim=1, largest=False).indices pred = torch.as_tensor(ytr)[idx].mean(1).numpy() return float(1 - ((yte - pred) ** 2).sum() / ((yte - yte.mean()) ** 2).sum()) def evaluate_similarity(model, Ztr, sh_tr, Zte, sh_te, k=5, n_pairs=20000, seed=0): """(1) kNN regression of summary stats from the embedding vs from the raw pyramid, on held-out countries (database = training pyramids); (2) Spearman between embedding cosine and -L1 distance.""" from scipy.stats import spearmanr etr, ete = embed(model, Ztr), embed(model, Zte) s_tr, s_te = summary_stats(sh_tr), summary_stats(sh_te) names = ["median_age", "share_under_15", "share_65_plus"] res = {"knn_r2_embedding": {}, "knn_r2_raw_pyramid": {}} raw_tr, raw_te = Ztr.reshape(len(Ztr), -1), Zte.reshape(len(Zte), -1) for j, nme in enumerate(names): res["knn_r2_embedding"][nme] = _knn_r2(etr, ete, s_tr[:, j], s_te[:, j], k, True) res["knn_r2_raw_pyramid"][nme] = _knn_r2(raw_tr, raw_te, s_tr[:, j], s_te[:, j], k, False) rng = np.random.default_rng(seed) i, j = rng.integers(0, len(Zte), n_pairs), rng.integers(0, len(Zte), n_pairs) cos = (ete[i] * ete[j]).sum(1) l1 = np.abs(_norm_shares(sh_te)[i] - _norm_shares(sh_te)[j]).sum((1, 2)) res["spearman_cosine_vs_neg_L1"] = float(spearmanr(cos, -l1).correlation) return res @torch.no_grad() def predict_heaping(model, Z, yrs, bs=1024): model.eval() dev = _dev(model) out = [] for i in range(0, len(Z), bs): x = torch.as_tensor(Z[i:i + bs]).to(dev) y = torch.as_tensor(year_feat(yrs[i:i + bs])).to(dev) out.append(torch.sigmoid(model(x, y)).cpu().numpy()) return np.concatenate(out) def _prf(pred, y): tp = float(((pred == 1) & (y == 1)).sum()) fp = float(((pred == 1) & (y == 0)).sum()) fn = float(((pred == 0) & (y == 1)).sum()) p, r = tp / max(tp + fp, 1), tp / max(tp + fn, 1) return {"precision": p, "recall": r, "f1": 2 * p * r / max(p + r, 1e-12)} def evaluate_heaping(model, norm, obs, y, yrs, lam): """Per-age detection on held-out countries vs a digit rule that flags every age ending in 0/5.""" from sklearn.metrics import roc_auc_score, average_precision_score prob = predict_heaping(model, norm.encode(obs), yrs) rule = np.zeros(N_AGES) rule[HEAP_AGES] = 1.0 rule = np.broadcast_to(rule, y.shape) clean = lam == 0 pred = (prob >= 0.5).astype(int) res = {"n_pyramids": int(len(y)), "positive_rate": float(y.mean()), "pyramidformer": {"auroc": float(roc_auc_score(y.ravel(), prob.ravel())), "avg_precision": float(average_precision_score(y.ravel(), prob.ravel())), **_prf(pred, y), "false_positive_rate_on_clean_pyramids": float(pred[clean].mean())}, "digit_rule_0_5": {**_prf(rule.astype(int), y), "false_positive_rate_on_clean_pyramids": float(rule[clean].mean())}} return res @torch.no_grad() def predict_forecast(model, norm, x0, years0, h, bs=2048): model.eval() dev = _dev(model) z0 = norm.encode(_norm_shares(x0)) yf = year_feat(np.broadcast_to(np.asarray(years0), (len(z0),))) out = [] for i in range(0, len(z0), bs): z = torch.as_tensor(z0[i:i + bs]).to(dev) y = torch.as_tensor(yf[i:i + bs]).to(dev) hh = torch.full((len(z),), int(h), dtype=torch.long, device=dev) out.append(model(z, y, h=hh).cpu().numpy()) return norm.decode(np.concatenate(out)) def eval_pairs(model, norm, x0, x1, years0, h): x0, x1 = _norm_shares(x0), _norm_shares(x1) pred = predict_forecast(model, norm, x0, years0, h) cs = cohort_shift(x0, h) m1 = summary_stats(x1)[:, 0] return {"n": int(len(x0)), "mae_permille": {"pyramidformer": mae_pm(pred, x1), "persistence": mae_pm(x0, x1), "cohort_shift": mae_pm(cs, x1)}, "median_age_abs_err_years": {"pyramidformer": float(np.abs(summary_stats(pred)[:, 0] - m1).mean()), "persistence": float(np.abs(summary_stats(x0)[:, 0] - m1).mean()), "cohort_shift": float(np.abs(summary_stats(cs)[:, 0] - m1).mean())}} def evaluate_forecast(model, norm, grid, idx, year0, hs=(1, 5, 10, 20, 25)): """All (origin year, origin year + h) pairs of the countries in idx (held-out countries).""" Zc = grid[idx] Y = Zc.shape[1] res = {} for h in hs: x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES) x1 = Zc[:, h:].reshape(-1, 2, N_AGES) yrs0 = np.tile(year0 + np.arange(Y - h), len(idx)) res[h] = eval_pairs(model, norm, x0, x1, yrs0, h) return res # ---------------------------------------------------------------------------- # Save / load # ---------------------------------------------------------------------------- def save_pretrained(path, model, norm, extra=None): from safetensors.torch import save_file os.makedirs(path, exist_ok=True) sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()} save_file(sd, os.path.join(path, "model.safetensors")) cfg = {"arch": model.cfg, "n_ages": N_AGES, "norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}} with open(os.path.join(path, "config.json"), "w") as f: json.dump(cfg, f, indent=2) def load_pretrained(path, device="cpu"): from safetensors.torch import load_file with open(os.path.join(path, "config.json")) as f: cfg = json.load(f) model = PyramidFormer(**cfg["arch"]) model.load_state_dict(load_file(os.path.join(path, "model.safetensors"))) model.to(device).eval() return model, Normaliser(**cfg["norm"])