pyramidformer-base / pyramidformer.py
SallySims's picture
Upload folder using huggingface_hub
d3f193c verified
Raw History Blame Contribute Delete
28.1 kB
"""
pyramidformer.py
================
PyramidFormer: a small transformer encoder over the 101 single-year age tokens of a
population pyramid (UN WPP), with task heads.
A pyramid is a (2, 101) array [male, female] x ages 0..100 (100 = 100+) of population
shares (sums to 1). Shares are log-transformed and standardised (Normaliser) before the model.
Tasks (PyramidFormer(task=...)):
base masked-age modelling (self-supervised pretraining): reconstruct hidden ages
similarity embedding of the pyramid shape (year is NOT an input): contrastive + auxiliary stats
regression + a distance-preserving term
heaping per-age logit: is this age inflated by digit preference (0/5 heaping)?
forecast predict the pyramid h years ahead (residual on the input pyramid), h = 1..25
"""
from __future__ import annotations
import json
import math
import os
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
N_AGES = 101
HEAP_AGES = list(range(5, 100, 5))
MAX_H = 25
# ----------------------------------------------------------------------------
# Data (UN WPP 2024 Excel files, single age, wide format)
# ----------------------------------------------------------------------------
def _read_wpp_xlsx(path, sheet):
"""Read one UN WPP 'single age' Excel sheet into a DataFrame: iso3, location, year, 0..100."""
xl = pd.ExcelFile(path)
names = xl.sheet_names
hit = [n for n in names if n.lower().strip() == sheet.lower()] or \
[n for n in names if sheet.lower() in n.lower()]
if not hit:
raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}")
raw = xl.parse(hit[0], header=None)
hdr = None
for i in range(min(len(raw), 60)):
cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()]
if "year" in cells and any(c.startswith("iso3") for c in cells):
hdr = i
break
if hdr is None:
raise ValueError(f"Could not find the header row in sheet '{hit[0]}' of {path}.")
cols = [str(v).strip() for v in raw.iloc[hdr].tolist()]
df = raw.iloc[hdr + 1:].copy()
df.columns = cols
low = {c.lower(): c for c in cols}
c_iso = next(c for c in cols if c.lower().startswith("iso3"))
c_year = low["year"]
c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso)
c_typ = low.get("type")
age_cols = {}
for c in cols:
t = c.replace(".0", "").strip()
if t.isdigit() and 0 <= int(t) <= 100:
age_cols[int(t)] = c
elif t.startswith("100"):
age_cols[100] = c
missing = [a for a in range(N_AGES) if a not in age_cols]
if missing:
raise ValueError(f"Age columns {missing[:5]}... not found in '{hit[0]}'; is this the SINGLE AGE file?")
if c_typ is not None:
df = df[df[c_typ].astype(str).str.startswith("Country")]
out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str),
"year": pd.to_numeric(df[c_year], errors="coerce")})
ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy()
for a in range(N_AGES)}, index=out.index)
out = pd.concat([out, ages_df], axis=1)
out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna()
out["year"] = out["year"].astype(int)
return out
def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023):
"""Returns (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands)."""
m = _read_wpp_xlsx(male_path, sheet)
f = _read_wpp_xlsx(female_path, sheet)
ages = list(range(N_AGES))
m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"])
f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"])
common = m.index.intersection(f.index).sort_values()
if len(common) == 0:
raise ValueError("No overlapping (country, year) rows between the male and female files.")
counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1)
meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(),
"year": [y for _, y in common]})
return meta, counts
def _norm_shares(a):
a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES)
return a / a.sum(axis=(1, 2), keepdims=True)
def counts_to_shares(counts):
counts = np.asarray(counts, dtype=np.float64)
return counts / counts.sum(axis=(-2, -1), keepdims=True)
def split_by_country(meta, val_frac=0.10, test_frac=0.10, seed=0):
"""Country-level split: no country appears in more than one split."""
rng = np.random.default_rng(seed)
iso = np.array(sorted(meta["iso3"].unique()))
rng.shuffle(iso)
n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac)
test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val]
is_test = meta["iso3"].isin(test_iso).to_numpy()
is_val = meta["iso3"].isin(val_iso).to_numpy()
return ~(is_test | is_val), is_val, is_test
def build_grid(meta, shares):
"""Arrange pyramids as (country, year, 2, 101). Returns (iso list, years array, grid of shares)."""
iso = sorted(meta["iso3"].unique())
yrs = np.sort(meta["year"].unique())
assert len(yrs) == yrs[-1] - yrs[0] + 1, "years are not contiguous"
ci = pd.Categorical(meta["iso3"], categories=iso).codes
yi = (meta["year"].to_numpy() - yrs[0]).astype(int)
grid = np.full((len(iso), len(yrs), 2, N_AGES), np.nan)
grid[ci, yi] = _norm_shares(shares)
return iso, yrs, grid
def summary_stats(shares):
"""(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females)."""
s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES)
tot = s.sum(axis=1)
tot = tot / tot.sum(axis=-1, keepdims=True)
cdf = np.cumsum(tot, axis=-1)
idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1)
rows = np.arange(len(s))
prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0)
frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12)
med = idx + np.clip(frac, 0.0, 1.0)
u15 = tot[:, :15].sum(-1)
p65 = tot[:, 65:].sum(-1)
sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12)
return np.stack([med, u15, p65, sr], axis=-1)
def stat_targets(shares):
"""Standardised (median age, share <15, share 65+) -> (n, 3) float32; auxiliary targets for the embedding."""
st = summary_stats(np.asarray(shares).reshape(-1, 2, N_AGES))
return np.stack([(st[:, 0] - 30) / 15, (st[:, 1] - 0.30) / 0.15, (st[:, 2] - 0.10) / 0.10], -1).astype(np.float32)
def year_feat(years):
return ((np.asarray(years, dtype=np.float32) - 1985.0) / 50.0).astype(np.float32)
class Normaliser:
"""log(share * 2 * N_AGES + eps), standardised with global mean / std."""
def __init__(self, mu=0.0, sigma=1.0, eps=1e-5):
self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps)
def _log(self, shares):
return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps)
def fit(self, shares):
z = self._log(shares)
self.mu, self.sigma = float(z.mean()), float(z.std())
return self
def encode(self, shares):
return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32)
def decode(self, x):
x = np.asarray(x, dtype=np.float64)
s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES)
return s / s.sum(axis=(-2, -1), keepdims=True)
# ----------------------------------------------------------------------------
# Synthetic age heaping (with per-age labels) and classical baselines
# ----------------------------------------------------------------------------
def heap_pyramids(shares, lam, rng, p_active=0.6, noise_sd=0.03, thr=0.05):
"""Digit preference at multiples of 5. Each pyramid gets its own random subset of 'active'
heaps with random strengths, so the pattern is not identical across pyramids.
A fraction lam*strength of people within +-2 years of a multiple of 5 report that multiple.
Returns (observed shares, labels (n, 101)): label = 1 where the age's count is inflated by > thr."""
truth = _norm_shares(shares)
out = truth.copy()
n = len(out)
lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (n,))
act = (rng.random((n, len(HEAP_AGES))) < p_active) * rng.uniform(0.4, 1.6, (n, len(HEAP_AGES)))
for j, h in enumerate(HEAP_AGES):
boost = 1.6 if h % 10 == 0 else 1.0
for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)):
frac = np.clip(lam * act[:, j] * boost * w, 0, 0.9)[:, None]
moved = out[:, :, h + d] * frac
out[:, :, h + d] -= moved
out[:, :, h] += moved
t, o = truth.sum(1), out.sum(1)
label = ((o / np.maximum(t, 1e-7) - 1.0) > thr) & (t > 1e-7)
out = out * np.exp(rng.normal(0.0, noise_sd, out.shape))
out = out / out.sum(axis=(1, 2), keepdims=True)
return out, label.astype(np.float32)
def make_heaping_set(shares, years, rng, copies=4, lam_range=(0.05, 0.35), p_clean=0.2):
"""Returns observed shares, per-age labels, years, lam (0 = clean pyramid)."""
base = np.repeat(_norm_shares(shares), copies, axis=0)
yrs = np.repeat(np.asarray(years), copies)
lam = rng.uniform(*lam_range, len(base))
lam[rng.random(len(base)) < p_clean] = 0.0
obs, y = heap_pyramids(base, lam, rng)
return obs, y, yrs, lam
def whipple(shares):
"""Whipple index (both sexes), ages 23-62. ~100 = no preference."""
tot = _norm_shares(shares).sum(axis=1)
ages = np.arange(23, 63)
return 500.0 * tot[:, ages[ages % 5 == 0]].sum(-1) / tot[:, 23:63].sum(-1)
def cohort_shift(shares, h):
"""Classical forecast baseline: age everyone by h years (no mortality), new cohorts = current age-0 size."""
s = _norm_shares(shares)
out = np.zeros_like(s)
for a in range(N_AGES):
out[..., min(a + h, N_AGES - 1)] += s[..., a]
out[..., :h] += s[..., :1]
return out / out.sum(axis=(-2, -1), keepdims=True)
def mae_pm(a, b):
"""Mean absolute error of shares, in per mille of total population."""
return float(np.abs(np.asarray(a) - np.asarray(b)).mean() * 1e3)
# ----------------------------------------------------------------------------
# Model
# ----------------------------------------------------------------------------
class PyramidFormer(nn.Module):
def __init__(self, task="base", d=128, depth=4, heads=4, ff=512, dropout=0.1, max_h=MAX_H, proj_dim=64):
super().__init__()
self.cfg = dict(task=task, d=d, depth=depth, heads=heads, ff=ff, dropout=dropout,
max_h=max_h, proj_dim=proj_dim)
self.task = task
self.inp = nn.Linear(2, d) # (male, female) z-values per age
self.age = nn.Parameter(torch.randn(1, N_AGES, d) * 0.02) # learned age embedding
self.year = nn.Sequential(nn.Linear(1, d), nn.SiLU(), nn.Linear(d, d))
self.mask_tok = nn.Parameter(torch.zeros(1, 1, d))
layer = nn.TransformerEncoderLayer(d_model=d, nhead=heads, dim_feedforward=ff, dropout=dropout,
activation="gelu", batch_first=True, norm_first=True)
self.enc = nn.TransformerEncoder(layer, num_layers=depth, enable_nested_tensor=False)
self.norm = nn.LayerNorm(d)
if task == "base":
self.recon = nn.Linear(d, 2)
elif task == "similarity":
self.proj = nn.Sequential(nn.Linear(d, d), nn.GELU(), nn.Linear(d, proj_dim))
self.aux = nn.Linear(proj_dim, 3) # auxiliary: standardised summary stats
elif task == "heaping":
self.tok = nn.Sequential(nn.Linear(d, d // 2), nn.GELU(), nn.Linear(d // 2, 1))
elif task == "forecast":
self.hemb = nn.Embedding(max_h + 1, d)
self.delta = nn.Linear(d, 2)
nn.init.zeros_(self.delta.weight) # start as persistence
nn.init.zeros_(self.delta.bias)
else:
raise ValueError(f"unknown task {task}")
def encode(self, x, year, mask=None, h=None):
t = self.inp(x.transpose(1, 2)) # (B, 101, d)
if mask is not None:
t = torch.where(mask[..., None], self.mask_tok.expand_as(t), t)
cond = self.year(year[:, None]) # (B, d)
if h is not None:
cond = cond + self.hemb(h)
t = t + self.age + cond[:, None, :]
return self.norm(self.enc(t))
def aux_stats(self, emb):
return self.aux(emb)
def forward(self, x, year, mask=None, h=None):
"""x: (B, 2, 101) z-encoded; year: (B,) from year_feat; mask: (B, 101) bool; h: (B,) long."""
if self.task == "similarity":
year = torch.zeros_like(year) # embedding must not depend on year
z = self.encode(x, year, mask, h)
if self.task == "base":
return self.recon(z).transpose(1, 2) # (B, 2, 101)
if self.task == "similarity":
return F.normalize(self.proj(z.mean(1)), dim=-1) # (B, proj_dim)
if self.task == "heaping":
return self.tok(z).squeeze(-1) # (B, 101) logits
return x + self.delta(z).transpose(1, 2) # forecast (B, 2, 101)
def init_from_base(model, base):
"""Copy the pretrained backbone into a task model (task heads stay freshly initialised)."""
sd = {k: v for k, v in base.state_dict().items() if not k.startswith("recon.")}
res = model.load_state_dict(sd, strict=False)
print("backbone loaded; new (head) params:", sorted({k.split('.')[0] for k in res.missing_keys}),
"| unexpected:", res.unexpected_keys)
return model
# ----------------------------------------------------------------------------
# Training
# ----------------------------------------------------------------------------
def fit(model, loss_fn, steps, lr=5e-4, wd=1e-2, log_every=500, val_fn=None, device=None, seed=0):
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
torch.manual_seed(seed)
model.to(device).train()
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)
history, run = [], []
for step in range(steps):
lr_now = lr * min(1.0, (step + 1) / 300) * 0.5 * (1 + math.cos(math.pi * step / steps))
for g in opt.param_groups:
g["lr"] = lr_now
loss = loss_fn(model, device)
opt.zero_grad(set_to_none=True)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
run.append(loss.item())
if (step + 1) % log_every == 0 or step + 1 == steps:
rec = {"step": step + 1, "train_loss": float(np.mean(run))}
run = []
if val_fn is not None:
model.eval()
with torch.no_grad():
rec["val_loss"] = float(val_fn(model, device))
model.train()
history.append(rec)
print(rec)
model.eval()
return history
def make_mask(B, device, p=0.3, gen=None):
"""Half the batch: random ages hidden; other half: one contiguous span of 5-20 ages hidden."""
rnd = torch.rand(B, N_AGES, device=device, generator=gen) < p
use_span = torch.rand(B, device=device, generator=gen) < 0.5
L = torch.randint(5, 21, (B, 1), device=device, generator=gen)
st = (torch.rand(B, 1, device=device, generator=gen) * (N_AGES - L)).long()
ar = torch.arange(N_AGES, device=device)[None]
span = (ar >= st) & (ar < st + L)
return torch.where(use_span[:, None], span, rnd)
def mam_loss_fn(X, Y, bs=256, p=0.3):
def f(model, device):
idx = torch.randint(0, X.shape[0], (bs,), device=device)
x, yr = X[idx], Y[idx]
m = make_mask(bs, device, p)
pred = model(x, yr, mask=m)
return ((pred - x) ** 2).mean(1)[m].mean()
return f
def mam_val_fn(X, Y, bs=512):
def f(model, device):
g = torch.Generator(device=device).manual_seed(123)
tot, n = 0.0, 0
for i in range(0, X.shape[0], bs):
x, yr = X[i:i + bs], Y[i:i + bs]
m = make_mask(len(x), device, 0.3, gen=g)
pred = model(x, yr, mask=m)
tot += ((pred - x) ** 2).mean(1)[m].sum().item()
n += int(m.sum().item())
return tot / max(n, 1)
return f
def sim_loss_fn(Z, S, c_idx, bs=256, tau=0.1, noise=0.02, aux_w=1.0, geo_w=2.0):
"""Z: (C, Y, 2, 101) z-encoded grid; S: (C, Y, 3) standardised summary stats; c_idx: training countries.
1. InfoNCE: positive = the same country 1-2 years earlier/later; same-country pairs are never negatives.
2. aux_w: the embedding must predict median age, share <15, share 65+ (linear head).
3. geo_w: cosine similarity of two pyramids in the batch should follow their L1 distance
(counteracts InfoNCE pushing apart different countries that have the same shape)."""
Y = Z.shape[1]
offs = torch.tensor([-2, -1, 1, 2])
def f(model, device):
o = offs.to(device)
c = c_idx[torch.randint(len(c_idx), (bs,), device=device)]
t = torch.randint(0, Y, (bs,), device=device)
t2 = (t + o[torch.randint(4, (bs,), device=device)]).clamp(0, Y - 1)
a0, b0 = Z[c, t], Z[c, t2]
a = a0 + noise * torch.randn_like(a0)
b = b0 + noise * torch.randn_like(b0)
zero = torch.zeros(bs, device=device)
ea, eb = model(a, zero), model(b, zero)
logits = ea @ eb.T / tau
same = (c[:, None] == c[None, :]) & ~torch.eye(bs, dtype=torch.bool, device=device)
logits = logits.masked_fill(same, -1e9)
lab = torch.arange(bs, device=device)
loss = 0.5 * (F.cross_entropy(logits, lab) + F.cross_entropy(logits.T, lab))
if aux_w > 0:
loss = loss + aux_w * 0.5 * (F.mse_loss(model.aux_stats(ea), S[c, t]) +
F.mse_loss(model.aux_stats(eb), S[c, t2]))
if geo_w > 0:
with torch.no_grad():
D = torch.cdist(a0.flatten(1), a0.flatten(1), p=1) / a0[0].numel()
T = torch.exp(-D / D.median().clamp(min=1e-6))
loss = loss + geo_w * F.mse_loss(ea @ ea.T, T)
return loss
return f
def heap_loss_fn(X, Yl, yrs, bs=256, pos_weight=3.0):
def f(model, device):
idx = torch.randint(0, X.shape[0], (bs,), device=device)
logits = model(X[idx], yrs[idx])
pw = torch.tensor(pos_weight, device=device)
return F.binary_cross_entropy_with_logits(logits, Yl[idx], pos_weight=pw)
return f
def heap_val_fn(X, Yl, yrs, bs=1024, pos_weight=3.0):
def f(model, device):
pw = torch.tensor(pos_weight, device=device)
tot, n = 0.0, 0
for i in range(0, X.shape[0], bs):
lg = model(X[i:i + bs], yrs[i:i + bs])
tot += F.binary_cross_entropy_with_logits(lg, Yl[i:i + bs], pos_weight=pw, reduction="sum").item()
n += lg.numel()
return tot / n
return f
def forecast_loss_fn(Z, c_idx, year0, max_h=MAX_H, bs=256):
Y = Z.shape[1]
def f(model, device):
c = c_idx[torch.randint(len(c_idx), (bs,), device=device)]
h = torch.randint(1, max_h + 1, (bs,), device=device)
t0 = (torch.rand(bs, device=device) * (Y - h)).long() # guarantees t0 + h <= Y - 1
yr = (year0 + t0.float() - 1985.0) / 50.0
return F.mse_loss(model(Z[c, t0], yr, h=h), Z[c, t0 + h])
return f
def forecast_val_fn(Z, c_idx, year0, hs=(1, 5, 10, 20, 25)):
Y = Z.shape[1]
def f(model, device):
Zc = Z[c_idx]
losses = []
for h in hs:
x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES)
x1 = Zc[:, h:].reshape(-1, 2, N_AGES)
yr = ((year0 + torch.arange(Y - h, device=device).float() - 1985.0) / 50.0).repeat(len(c_idx))
hh = torch.full((len(x0),), h, dtype=torch.long, device=device)
losses.append(F.mse_loss(model(x0, yr, h=hh), x1).item())
return float(np.mean(losses))
return f
# ----------------------------------------------------------------------------
# Inference helpers and evaluation
# ----------------------------------------------------------------------------
def _dev(model):
return next(model.parameters()).device
@torch.no_grad()
def embed(model, Z, bs=1024):
"""Z: z-encoded pyramids (n, 2, 101) -> L2-normalised embeddings (n, proj_dim)."""
model.eval()
dev = _dev(model)
out = []
for i in range(0, len(Z), bs):
x = torch.as_tensor(Z[i:i + bs]).to(dev)
out.append(model(x, torch.zeros(len(x), device=dev)).cpu())
return torch.cat(out).numpy()
def most_similar(model, norm, query_shares, E_db, meta, k=5, exclude_iso=None, per_country=True):
"""Nearest pyramids in a database of embeddings E_db (rows aligned with meta).
per_country=True keeps only the best-matching year of each country, so results are k different countries."""
q = embed(model, norm.encode(_norm_shares(query_shares)))
S = q @ E_db.T
iso = meta["iso3"].to_numpy()
if exclude_iso is not None:
S[:, iso == exclude_iso] = -1.0
out = []
for j in range(len(q)):
keep, seen = [], set()
for i in np.argsort(-S[j]):
if per_country and iso[i] in seen:
continue
seen.add(iso[i])
keep.append(int(i))
if len(keep) == k:
break
out.append(meta.iloc[keep][["iso3", "location", "year"]].assign(cosine=S[j, keep]).reset_index(drop=True))
return out
def _knn_r2(Ftr, Fte, ytr, yte, k=5, cosine=True):
Ft, Fe = torch.as_tensor(Ftr), torch.as_tensor(Fte)
if cosine:
idx = (Fe @ Ft.T).topk(k, dim=1).indices
else:
idx = torch.cdist(Fe, Ft).topk(k, dim=1, largest=False).indices
pred = torch.as_tensor(ytr)[idx].mean(1).numpy()
return float(1 - ((yte - pred) ** 2).sum() / ((yte - yte.mean()) ** 2).sum())
def evaluate_similarity(model, Ztr, sh_tr, Zte, sh_te, k=5, n_pairs=20000, seed=0):
"""(1) kNN regression of summary stats from the embedding vs from the raw pyramid, on held-out
countries (database = training pyramids); (2) Spearman between embedding cosine and -L1 distance."""
from scipy.stats import spearmanr
etr, ete = embed(model, Ztr), embed(model, Zte)
s_tr, s_te = summary_stats(sh_tr), summary_stats(sh_te)
names = ["median_age", "share_under_15", "share_65_plus"]
res = {"knn_r2_embedding": {}, "knn_r2_raw_pyramid": {}}
raw_tr, raw_te = Ztr.reshape(len(Ztr), -1), Zte.reshape(len(Zte), -1)
for j, nme in enumerate(names):
res["knn_r2_embedding"][nme] = _knn_r2(etr, ete, s_tr[:, j], s_te[:, j], k, True)
res["knn_r2_raw_pyramid"][nme] = _knn_r2(raw_tr, raw_te, s_tr[:, j], s_te[:, j], k, False)
rng = np.random.default_rng(seed)
i, j = rng.integers(0, len(Zte), n_pairs), rng.integers(0, len(Zte), n_pairs)
cos = (ete[i] * ete[j]).sum(1)
l1 = np.abs(_norm_shares(sh_te)[i] - _norm_shares(sh_te)[j]).sum((1, 2))
res["spearman_cosine_vs_neg_L1"] = float(spearmanr(cos, -l1).correlation)
return res
@torch.no_grad()
def predict_heaping(model, Z, yrs, bs=1024):
model.eval()
dev = _dev(model)
out = []
for i in range(0, len(Z), bs):
x = torch.as_tensor(Z[i:i + bs]).to(dev)
y = torch.as_tensor(year_feat(yrs[i:i + bs])).to(dev)
out.append(torch.sigmoid(model(x, y)).cpu().numpy())
return np.concatenate(out)
def _prf(pred, y):
tp = float(((pred == 1) & (y == 1)).sum())
fp = float(((pred == 1) & (y == 0)).sum())
fn = float(((pred == 0) & (y == 1)).sum())
p, r = tp / max(tp + fp, 1), tp / max(tp + fn, 1)
return {"precision": p, "recall": r, "f1": 2 * p * r / max(p + r, 1e-12)}
def evaluate_heaping(model, norm, obs, y, yrs, lam):
"""Per-age detection on held-out countries vs a digit rule that flags every age ending in 0/5."""
from sklearn.metrics import roc_auc_score, average_precision_score
prob = predict_heaping(model, norm.encode(obs), yrs)
rule = np.zeros(N_AGES)
rule[HEAP_AGES] = 1.0
rule = np.broadcast_to(rule, y.shape)
clean = lam == 0
pred = (prob >= 0.5).astype(int)
res = {"n_pyramids": int(len(y)), "positive_rate": float(y.mean()),
"pyramidformer": {"auroc": float(roc_auc_score(y.ravel(), prob.ravel())),
"avg_precision": float(average_precision_score(y.ravel(), prob.ravel())),
**_prf(pred, y), "false_positive_rate_on_clean_pyramids": float(pred[clean].mean())},
"digit_rule_0_5": {**_prf(rule.astype(int), y),
"false_positive_rate_on_clean_pyramids": float(rule[clean].mean())}}
return res
@torch.no_grad()
def predict_forecast(model, norm, x0, years0, h, bs=2048):
model.eval()
dev = _dev(model)
z0 = norm.encode(_norm_shares(x0))
yf = year_feat(np.broadcast_to(np.asarray(years0), (len(z0),)))
out = []
for i in range(0, len(z0), bs):
z = torch.as_tensor(z0[i:i + bs]).to(dev)
y = torch.as_tensor(yf[i:i + bs]).to(dev)
hh = torch.full((len(z),), int(h), dtype=torch.long, device=dev)
out.append(model(z, y, h=hh).cpu().numpy())
return norm.decode(np.concatenate(out))
def eval_pairs(model, norm, x0, x1, years0, h):
x0, x1 = _norm_shares(x0), _norm_shares(x1)
pred = predict_forecast(model, norm, x0, years0, h)
cs = cohort_shift(x0, h)
m1 = summary_stats(x1)[:, 0]
return {"n": int(len(x0)),
"mae_permille": {"pyramidformer": mae_pm(pred, x1), "persistence": mae_pm(x0, x1),
"cohort_shift": mae_pm(cs, x1)},
"median_age_abs_err_years": {"pyramidformer": float(np.abs(summary_stats(pred)[:, 0] - m1).mean()),
"persistence": float(np.abs(summary_stats(x0)[:, 0] - m1).mean()),
"cohort_shift": float(np.abs(summary_stats(cs)[:, 0] - m1).mean())}}
def evaluate_forecast(model, norm, grid, idx, year0, hs=(1, 5, 10, 20, 25)):
"""All (origin year, origin year + h) pairs of the countries in idx (held-out countries)."""
Zc = grid[idx]
Y = Zc.shape[1]
res = {}
for h in hs:
x0 = Zc[:, :Y - h].reshape(-1, 2, N_AGES)
x1 = Zc[:, h:].reshape(-1, 2, N_AGES)
yrs0 = np.tile(year0 + np.arange(Y - h), len(idx))
res[h] = eval_pairs(model, norm, x0, x1, yrs0, h)
return res
# ----------------------------------------------------------------------------
# Save / load
# ----------------------------------------------------------------------------
def save_pretrained(path, model, norm, extra=None):
from safetensors.torch import save_file
os.makedirs(path, exist_ok=True)
sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()}
save_file(sd, os.path.join(path, "model.safetensors"))
cfg = {"arch": model.cfg, "n_ages": N_AGES,
"norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}}
with open(os.path.join(path, "config.json"), "w") as f:
json.dump(cfg, f, indent=2)
def load_pretrained(path, device="cpu"):
from safetensors.torch import load_file
with open(os.path.join(path, "config.json")) as f:
cfg = json.load(f)
model = PyramidFormer(**cfg["arch"])
model.load_state_dict(load_file(os.path.join(path, "model.safetensors")))
model.to(device).eval()
return model, Normaliser(**cfg["norm"])