pyramiddiff-wpp2024 / pyramiddiff.py
SallySims's picture
Upload pyramiddiff.py with huggingface_hub
8e5989b verified
Raw History Blame Contribute Delete
36.3 kB
"""
pyramiddiff.py
==============
PyramidDiff: a conditional diffusion model of single-age, sex-specific population
structure (age-sex pyramids), trained on UN World Population Prospects (WPP).
What it does
------------
1. generate : sample plausible pyramids given summary conditions
(median age, % under 15, % 65+, sex ratio, year) with uncertainty bands.
2. repair : denoise age-heaped / digit-preferenced pyramids (SDEdit-style partial
noising + denoising), then project back onto the observed totals of
5-year windows centred on the heaping digits (0 and 5).
3. inpaint : fill in missing ages (RePaint-style).
4. anomaly : score how implausible a pyramid is (denoising error).
Representation
--------------
A pyramid is a (2, 101) array: [male, female] x ages 0..100 (100 = 100+), expressed
as shares of the total population (sums to 1). Shares are log-transformed and
standardised before diffusion.
"""
from __future__ import annotations
import copy
import json
import math
import os
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
N_AGES = 101
PAD_LEN = 104 # padded length (divisible by 4) used inside the UNet
COND_DIM = 5
# Windows centred on the heaping digits: [0,3) [3,8) [8,13) ... [93,98) [98,101)
WINDOW_EDGES = [0] + list(range(3, 99, 5)) + [101]
# ----------------------------------------------------------------------------
# Data
# ----------------------------------------------------------------------------
def load_wpp(source: str, year_min: int = 1950, year_max: int = 2023):
"""Load a LOCAL UN WPP CSV (e.g. WPP2024_PopulationBySingleAgeSex_Medium_1950-2023.csv.gz).
Return (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands).
Column names are matched case-insensitively and the available columns are printed on failure."""
raw = pd.read_csv(source, low_memory=False)
low = {c.lower().strip(): c for c in raw.columns}
def pick(*names, required=True):
for n in names:
if n.lower() in low:
return low[n.lower()]
if required:
raise ValueError(f"Could not find any of {names} in the WPP file. Columns found: {list(raw.columns)}")
return None
c_iso = pick("ISO3_code", "ISO3", required=False)
c_loc = pick("Location", "Country", "Area")
c_typ = pick("LocTypeName", "LocType", required=False)
c_t = pick("Time", "Year")
c_age = pick("AgeGrpStart", "Age")
c_m, c_f = pick("PopMale", "Male"), pick("PopFemale", "Female")
df = raw
if c_typ is not None:
df = df[df[c_typ].astype(str).str.startswith("Country")]
key = c_iso if c_iso is not None else c_loc
df = df[df[key].notna() & df[c_t].between(year_min, year_max)]
df = df.sort_values([key, c_t, c_age])
size = df.groupby([key, c_t])[c_age].transform("size")
df = df[size == N_AGES]
if len(df) == 0:
raise ValueError("No complete single-age (0..100) blocks found; check you downloaded the "
"PopulationBySingleAgeSex file, not the 5-year-group file.")
ages = df[c_age].to_numpy().reshape(-1, N_AGES)
assert (ages == np.arange(N_AGES)).all(), "Age groups are not 0..100 in every block"
counts = (
df[[c_m, c_f]].to_numpy(dtype=np.float64)
.reshape(-1, N_AGES, 2).transpose(0, 2, 1)
)
meta = df.iloc[::N_AGES][[key, c_loc, c_t]].reset_index(drop=True)
meta.columns = ["iso3", "location", "year"]
return meta, counts
def _read_wpp_xlsx(path, sheet):
"""Read one UN WPP 'single age' Excel sheet (wide format: one column per age) into a DataFrame
with columns iso3, location, year, 0..100. The header row is located automatically."""
xl = pd.ExcelFile(path)
names = xl.sheet_names
hit = [n for n in names if n.lower().strip() == sheet.lower()] or \
[n for n in names if sheet.lower() in n.lower()]
if not hit:
raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}")
raw = xl.parse(hit[0], header=None)
hdr = None
for i in range(min(len(raw), 60)):
cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()]
if "year" in cells and any(c.startswith("iso3") for c in cells):
hdr = i
break
if hdr is None:
raise ValueError(f"Could not find the header row (with 'Year' and an ISO3 column) in sheet "
f"'{hit[0]}' of {path}.")
cols = [str(v).strip() for v in raw.iloc[hdr].tolist()]
df = raw.iloc[hdr + 1:].copy()
df.columns = cols
low = {c.lower(): c for c in cols}
c_iso = next(c for c in cols if c.lower().startswith("iso3"))
c_year = low["year"]
c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso)
c_typ = low.get("type")
age_cols = {}
for c in cols:
t = c.replace(".0", "").strip()
if t.isdigit() and 0 <= int(t) <= 100:
age_cols[int(t)] = c
elif t.startswith("100"):
age_cols[100] = c
missing = [a for a in range(N_AGES) if a not in age_cols]
if missing:
raise ValueError(f"Age columns {missing[:5]}... not found in sheet '{hit[0]}'; is this the "
f"SINGLE AGE file (not 5-year groups)?")
if c_typ is not None:
df = df[df[c_typ].astype(str).str.startswith("Country")]
out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str),
"year": pd.to_numeric(df[c_year], errors="coerce")})
ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy()
for a in range(N_AGES)}, index=out.index)
out = pd.concat([out, ages_df], axis=1)
out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna()
out["year"] = out["year"].astype(int)
return out
def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023):
"""Load the UN WPP Excel files 'Population by Single Age - Male / Female'.
Use sheet='Estimates' for 1950-2023 and sheet='Medium variant' for 2024-2100.
Returns the same (meta, counts) as load_wpp."""
m = _read_wpp_xlsx(male_path, sheet)
f = _read_wpp_xlsx(female_path, sheet)
ages = list(range(N_AGES))
m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"])
f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"])
common = m.index.intersection(f.index).sort_values()
if len(common) == 0:
raise ValueError("No overlapping (country, year) rows between the male and female files.")
counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1)
meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(),
"year": [y for _, y in common]})
return meta, counts
def counts_to_shares(counts):
counts = np.asarray(counts, dtype=np.float64)
return counts / counts.sum(axis=(-2, -1), keepdims=True)
def split_by_country(meta: pd.DataFrame, val_frac=0.10, test_frac=0.10, seed=0):
"""Country-level split so that no country appears in more than one split."""
rng = np.random.default_rng(seed)
iso = np.array(sorted(meta["iso3"].unique()))
rng.shuffle(iso)
n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac)
test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val]
is_test = meta["iso3"].isin(test_iso).to_numpy()
is_val = meta["iso3"].isin(val_iso).to_numpy()
is_train = ~(is_test | is_val)
return is_train, is_val, is_test
def summary_stats(shares):
"""(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females)."""
s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES)
tot = s.sum(axis=1)
tot = tot / tot.sum(axis=-1, keepdims=True)
cdf = np.cumsum(tot, axis=-1)
idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1)
rows = np.arange(len(s))
prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0)
frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12)
med = idx + np.clip(frac, 0.0, 1.0)
u15 = tot[:, :15].sum(-1)
p65 = tot[:, 65:].sum(-1)
sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12)
return np.stack([med, u15, p65, sr], axis=-1)
def build_cond(stats, years):
"""Conditioning vector (n, 5), roughly standardised."""
stats = np.asarray(stats, dtype=np.float64).reshape(-1, 4)
med, u15, p65, sr = stats.T
years = np.broadcast_to(np.asarray(years, dtype=np.float64), (len(stats),))
c = np.stack(
[(med - 30) / 15, (u15 - 0.30) / 0.15, (p65 - 0.10) / 0.10,
np.clip((sr - 100) / 15, -3, 3), (years - 1985) / 50], axis=1)
return c.astype(np.float32)
class Normaliser:
"""log(share * 2 * N_AGES + eps), standardised with global mean / std."""
def __init__(self, mu=0.0, sigma=1.0, eps=1e-5):
self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps)
def _log(self, shares):
return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps)
def fit(self, shares):
z = self._log(shares)
self.mu, self.sigma = float(z.mean()), float(z.std())
return self
def encode(self, shares):
return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32)
def decode(self, x):
x = np.asarray(x, dtype=np.float64)
s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES)
return s / s.sum(axis=(-2, -1), keepdims=True)
# ----------------------------------------------------------------------------
# Model: small conditional 1D UNet with FiLM conditioning and a bottleneck attention
# ----------------------------------------------------------------------------
class SinusoidalEmb(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
def forward(self, t):
half = self.dim // 2
freqs = torch.exp(-math.log(10000.0) * torch.arange(half, device=t.device) / half)
a = t.float()[:, None] * freqs[None]
return torch.cat([a.sin(), a.cos()], dim=-1)
class Block(nn.Module):
def __init__(self, cin, cout, emb_dim):
super().__init__()
self.n1 = nn.GroupNorm(8, cin)
self.c1 = nn.Conv1d(cin, cout, 5, padding=2)
self.n2 = nn.GroupNorm(8, cout)
self.c2 = nn.Conv1d(cout, cout, 5, padding=2)
self.film = nn.Linear(emb_dim, 2 * cout)
self.skip = nn.Conv1d(cin, cout, 1) if cin != cout else nn.Identity()
def forward(self, x, e):
h = self.c1(F.silu(self.n1(x)))
scale, shift = self.film(e).chunk(2, dim=1)
h = self.n2(h) * (1 + scale[..., None]) + shift[..., None]
h = self.c2(F.silu(h))
return h + self.skip(x)
class SelfAttn(nn.Module):
def __init__(self, c, heads):
super().__init__()
self.norm = nn.GroupNorm(8, c)
self.attn = nn.MultiheadAttention(c, heads, batch_first=True)
def forward(self, x):
h = self.norm(x).transpose(1, 2)
h, _ = self.attn(h, h, h, need_weights=False)
return x + h.transpose(1, 2)
class PyramidUNet(nn.Module):
def __init__(self, base=64, mults=(1, 2, 4), emb_dim=256, cond_dim=COND_DIM, heads=4):
super().__init__()
self.cfg = dict(base=base, mults=list(mults), emb_dim=emb_dim, cond_dim=cond_dim, heads=heads)
self.temb = nn.Sequential(SinusoidalEmb(emb_dim), nn.Linear(emb_dim, emb_dim),
nn.SiLU(), nn.Linear(emb_dim, emb_dim))
self.cemb = nn.Sequential(nn.Linear(cond_dim, emb_dim), nn.SiLU(), nn.Linear(emb_dim, emb_dim))
self.null = nn.Parameter(torch.zeros(emb_dim)) # "no condition" embedding (for CFG)
self.register_buffer("pos", torch.linspace(-1, 1, PAD_LEN)[None, None, :], persistent=False)
self.inp = nn.Conv1d(3, base, 5, padding=2) # male, female, age-position channel
chs = [base * m for m in mults]
self.down_blocks, self.downs = nn.ModuleList(), nn.ModuleList()
c = base
for i, ch in enumerate(chs):
self.down_blocks.append(nn.ModuleList([Block(c, ch, emb_dim), Block(ch, ch, emb_dim)]))
self.downs.append(nn.Conv1d(ch, ch, 3, stride=2, padding=1) if i < len(chs) - 1 else nn.Identity())
c = ch
self.mid1, self.attn, self.mid2 = Block(c, c, emb_dim), SelfAttn(c, heads), Block(c, c, emb_dim)
self.up_blocks, self.ups = nn.ModuleList(), nn.ModuleList()
cur = chs[-1]
for i in reversed(range(len(chs))):
self.up_blocks.append(nn.ModuleList([Block(cur + chs[i], chs[i], emb_dim),
Block(chs[i], chs[i], emb_dim)]))
self.ups.append(
nn.Sequential(nn.Upsample(scale_factor=2, mode="nearest"),
nn.Conv1d(chs[i], chs[i], 3, padding=1)) if i > 0 else nn.Identity())
cur = chs[i]
self.out_norm = nn.GroupNorm(8, base)
self.out = nn.Conv1d(base, 2, 5, padding=2)
def forward(self, x, t, cond=None, drop=None):
B = x.shape[0]
e = self.temb(t)
if cond is None:
ce = self.null.expand(B, -1)
else:
ce = self.cemb(cond)
if drop is not None:
ce = torch.where(drop[:, None], self.null.expand(B, -1), ce)
e = e + ce
h = F.pad(x, (0, PAD_LEN - N_AGES), mode="replicate")
h = torch.cat([h, self.pos.expand(B, -1, -1)], dim=1)
h = self.inp(h)
skips = []
for blocks, down in zip(self.down_blocks, self.downs):
for b in blocks:
h = b(h, e)
skips.append(h)
h = down(h)
h = self.mid2(self.attn(self.mid1(h, e)), e)
for blocks, up in zip(self.up_blocks, self.ups):
h = torch.cat([h, skips.pop()], dim=1)
for b in blocks:
h = b(h, e)
h = up(h)
return self.out(F.silu(self.out_norm(h)))[..., :N_AGES]
class EMA:
def __init__(self, model, decay=0.999):
self.decay = decay
self.shadow = copy.deepcopy(model).eval()
for p in self.shadow.parameters():
p.requires_grad_(False)
@torch.no_grad()
def update(self, model, step=None):
d = self.decay if step is None else min(self.decay, (1 + step) / (10 + step)) # warm-up
for ps, p in zip(self.shadow.parameters(), model.parameters()):
ps.mul_(d).add_(p.detach(), alpha=1 - d)
# ----------------------------------------------------------------------------
# Diffusion
# ----------------------------------------------------------------------------
class Diffusion:
def __init__(self, T=500, device="cpu"):
self.T = T
s = 0.008
t = torch.arange(T + 1, dtype=torch.float64) / T
f = torch.cos((t + s) / (1 + s) * math.pi / 2) ** 2
abar = f / f[0]
betas = (1 - abar[1:] / abar[:-1]).clamp(1e-5, 0.999)
self.abar = torch.cumprod(1 - betas, dim=0).float()
self.to(device)
def to(self, device):
self.abar = self.abar.to(device)
return self
def q_sample(self, x0, t, noise=None):
noise = torch.randn_like(x0) if noise is None else noise
a = self.abar[t].view(-1, 1, 1)
return a.sqrt() * x0 + (1 - a).sqrt() * noise, noise
@torch.no_grad()
def sample(model, diff, cond=None, n=None, n_steps=50, guidance=1.5, eta=0.0,
x_init=None, t_start=None, known=None, known_mask=None, clip=6.0, x0_hook=None):
"""DDIM sampler with classifier-free guidance.
- plain generation : x_init=None
- SDEdit repair : x_init=<clean-ish observed pyramid>, t_start=<noise level>
- inpainting : known=<values>, known_mask=<1 where observed> (RePaint-style)
"""
model.eval()
device = next(model.parameters()).device
diff.to(device)
t_start = diff.T - 1 if t_start is None else int(t_start)
if x_init is None:
B = n if cond is None else cond.shape[0]
if B is None:
B = known.shape[0]
x = torch.randn(B, 2, N_AGES, device=device)
else:
B = x_init.shape[0]
x = diff.q_sample(x_init, torch.full((B,), t_start, device=device, dtype=torch.long))[0]
def predict(xt, t):
tt = torch.full((B,), int(t), device=device, dtype=torch.long)
if cond is None:
return model(xt, tt, None)
ec = model(xt, tt, cond)
if guidance == 1.0:
return ec
eu = model(xt, tt, None)
return eu + guidance * (ec - eu)
ts = []
for t in np.round(np.linspace(t_start, 0, n_steps)).astype(int):
if not ts or t != ts[-1]:
ts.append(int(t))
for i, t in enumerate(ts):
t_prev = ts[i + 1] if i + 1 < len(ts) else -1
eps = predict(x, t)
ab = diff.abar[t]
ab_prev = diff.abar[t_prev] if t_prev >= 0 else torch.tensor(1.0, device=device)
x0 = ((x - (1 - ab).sqrt() * eps) / ab.sqrt()).clamp(-clip, clip)
if x0_hook is not None: # e.g. enforce known window totals on the clean estimate
x0 = x0_hook(x0)
eps = (x - ab.sqrt() * x0) / (1 - ab).sqrt()
sigma = eta * ((1 - ab_prev) / (1 - ab)).sqrt() * (1 - ab / ab_prev).clamp(min=0).sqrt()
direction = (1 - ab_prev - sigma ** 2).clamp(min=0).sqrt() * eps
x = ab_prev.sqrt() * x0 + direction + (sigma * torch.randn_like(x) if eta > 0 else 0.0)
if known is not None:
if t_prev >= 0:
kn = diff.q_sample(known, torch.full((B,), t_prev, device=device, dtype=torch.long))[0]
else:
kn = known
x = known_mask * kn + (1 - known_mask) * x
return x
# ----------------------------------------------------------------------------
# Training
# ----------------------------------------------------------------------------
@torch.no_grad()
def val_loss(model, diff, X, C, seed=123):
g = torch.Generator(device=X.device).manual_seed(seed)
losses = []
for t_val in (25, 75, 150, 250, 400):
t = torch.full((X.shape[0],), t_val, device=X.device, dtype=torch.long)
noise = torch.randn(X.shape, generator=g, device=X.device)
xt, _ = diff.q_sample(X, t, noise)
losses.append(F.mse_loss(model(xt, t, C), noise).item())
return float(np.mean(losses))
def train(model, diff, x, c, x_val=None, c_val=None, steps=12000, bs=256, lr=3e-4,
p_uncond=0.1, ema_decay=0.999, log_every=500, device=None, seed=0):
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
torch.manual_seed(seed)
model.to(device).train()
diff.to(device)
ema = EMA(model, ema_decay)
ema.shadow.to(device)
X, C = torch.as_tensor(x).to(device), torch.as_tensor(c).to(device)
Xv = torch.as_tensor(x_val).to(device) if x_val is not None else None
Cv = torch.as_tensor(c_val).to(device) if c_val is not None else None
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)
history, run = [], []
for step in range(steps):
lr_now = lr * min(1.0, (step + 1) / 500) * 0.5 * (1 + math.cos(math.pi * step / steps))
for g in opt.param_groups:
g["lr"] = lr_now
idx = torch.randint(0, X.shape[0], (bs,), device=device)
t = torch.randint(0, diff.T, (bs,), device=device)
xt, noise = diff.q_sample(X[idx], t)
drop = torch.rand(bs, device=device) < p_uncond
loss = F.mse_loss(model(xt, t, C[idx], drop), noise)
opt.zero_grad(set_to_none=True)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
ema.update(model, step)
run.append(loss.item())
if (step + 1) % log_every == 0 or step + 1 == steps:
rec = {"step": step + 1, "train_loss": float(np.mean(run))}
run = []
if Xv is not None:
rec["val_loss_ema"] = val_loss(ema.shadow, diff, Xv, Cv)
history.append(rec)
print(rec)
model.eval()
return ema, history
# ----------------------------------------------------------------------------
# Age heaping: simulator, index, repair
# ----------------------------------------------------------------------------
def simulate_heaping(shares, lam, rng=None, digit0_boost=1.6, noise_sd=0.03):
"""Mimic digit preference. A fraction `lam` (per pyramid) of people aged within +-2 of a
multiple of 5 report that multiple instead (stronger for multiples of 10), plus mild
multiplicative roster noise. shares: (n, 2, 101); lam: scalar or (n,)."""
rng = np.random.default_rng() if rng is None else rng
out = np.array(shares, dtype=np.float64).reshape(-1, 2, N_AGES)
lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (len(out),))
for h in range(5, 100, 5):
boost = digit0_boost if h % 10 == 0 else 1.0
for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)):
frac = np.clip(lam * boost * w, 0, 0.9)[:, None]
moved = out[:, :, h + d] * frac
out[:, :, h + d] -= moved
out[:, :, h] += moved
out *= np.exp(rng.normal(0.0, noise_sd, out.shape))
return out / out.sum(axis=(1, 2), keepdims=True)
def whipple(shares):
"""Whipple index (both sexes), ages 23-62. ~100 = no preference, 500 = only 0/5."""
tot = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES).sum(axis=1)
ages = np.arange(23, 63)
num = tot[:, ages[ages % 5 == 0]].sum(-1)
den = tot[:, 23:63].sum(-1)
return 500.0 * num / den
def project_to_windows(rep, obs):
"""Rescale `rep` so that totals of windows centred on heaping digits match `obs`
(heaping moves people within +-2 years, so these totals are approximately conserved)."""
rep = np.asarray(rep, dtype=np.float64)
obs = np.asarray(obs, dtype=np.float64)
out = rep.copy()
for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]):
r = rep[..., a:b].sum(-1, keepdims=True)
o = obs[..., a:b].sum(-1, keepdims=True)
out[..., a:b] = rep[..., a:b] * (o / np.maximum(r, 1e-12))
return out
def _norm_shares(a):
a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES)
return a / a.sum(axis=(1, 2), keepdims=True)
def _decode_t(x, norm):
s = (torch.exp(x * norm.sigma + norm.mu) - norm.eps).clamp(min=0.0) / (2 * N_AGES)
return s / s.sum(dim=(-2, -1), keepdim=True)
def _encode_t(s, norm):
return (torch.log(s * 2 * N_AGES + norm.eps) - norm.mu) / norm.sigma
def _project_t(rep, obs):
out = rep.clone()
for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]):
r = rep[..., a:b].sum(-1, keepdim=True)
o = obs[..., a:b].sum(-1, keepdim=True)
out[..., a:b] = rep[..., a:b] * (o / r.clamp(min=1e-12))
return out
def pchip_baseline(obs):
"""Classical window-total-preserving smoother: monotone (PCHIP) interpolation of the cumulative
population at the window edges, differenced back to single ages. Heaping cannot move the
window totals much, so this removes digit preference without a learned model."""
from scipy.interpolate import PchipInterpolator
obs = _norm_shares(obs)
edges = np.array(WINDOW_EDGES)
tot = np.add.reduceat(obs, edges[:-1], axis=-1) # (n, 2, n_windows)
cum = np.concatenate([np.zeros(tot.shape[:-1] + (1,)), np.cumsum(tot, axis=-1)], axis=-1)
f = PchipInterpolator(edges, cum, axis=-1)
out = np.clip(np.diff(f(np.arange(N_AGES + 1)), axis=-1), 0.0, None)
return out / out.sum(axis=(1, 2), keepdims=True)
def repair(model, diff, norm, obs_shares, years, t_start=200, n_steps=40, n_samples=4,
use_cond=True, project="each_step", init="pchip", seed=0):
"""Repair heaped pyramids. Returns shares (n, 2, 101).
init : "pchip" starts from the classical window-total-preserving smoothing (in-distribution
input for the model); "obs" starts from the raw heaped pyramid.
project : "each_step" re-imposes the observed window totals on the clean estimate at every
denoising step (the model only decides the within-window shape); "post" projects once
at the end; "none" does not project.
"""
device = next(model.parameters()).device
obs = _norm_shares(obs_shares)
start = pchip_baseline(obs) if init == "pchip" else obs
x_start = torch.as_tensor(norm.encode(start)).to(device)
cond = None
if use_cond:
cond = torch.as_tensor(build_cond(summary_stats(obs), years)).to(device)
hook = None
if project == "each_step":
obs_t = torch.as_tensor(obs, dtype=torch.float32, device=device)
def hook(x0):
return _encode_t(_project_t(_decode_t(x0, norm), obs_t), norm).clamp(-6.0, 6.0)
torch.manual_seed(seed)
outs = []
for _ in range(n_samples):
x = sample(model, diff, cond, n_steps=n_steps, guidance=1.0, eta=0.0,
x_init=x_start, t_start=t_start, x0_hook=hook)
outs.append(norm.decode(x.cpu().numpy()))
rep = np.mean(outs, axis=0)
if project in ("each_step", "post"):
rep = project_to_windows(rep, obs)
return rep / rep.sum(axis=(1, 2), keepdims=True)
def inpaint(model, diff, norm, obs_shares, observed_mask, years, n_steps=60, guidance=1.5, seed=0):
"""Fill unobserved ages. observed_mask: (2,101) or (n,2,101) bool, True = observed.
Values at unobserved positions in obs_shares are ignored (use any positive number)."""
device = next(model.parameters()).device
obs = np.clip(_norm_shares(obs_shares), 1e-9, None)
mask = np.broadcast_to(np.asarray(observed_mask, dtype=bool), obs.shape)
known = torch.as_tensor(norm.encode(obs)).to(device)
m = torch.as_tensor(mask.astype(np.float32)).to(device)
# conditioning from the observed part only: fill the gaps with the mean observed share
filled = np.where(mask, obs, obs[mask].mean())
cond = torch.as_tensor(build_cond(summary_stats(filled), years)).to(device)
torch.manual_seed(seed)
x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.3,
known=known, known_mask=m)
return norm.decode(x.cpu().numpy())
@torch.no_grad()
def anomaly_score(model, diff, x, cond, ts=(20, 40, 60, 80, 120), seed=0):
"""Mean noise-prediction error at low noise levels; higher = less plausible pyramid."""
model.eval()
diff.to(x.device)
g = torch.Generator(device=x.device).manual_seed(seed)
errs = []
for t_val in ts:
t = torch.full((x.shape[0],), t_val, device=x.device, dtype=torch.long)
noise = torch.randn(x.shape, generator=g, device=x.device)
xt, _ = diff.q_sample(x, t, noise)
errs.append(((model(xt, t, cond) - noise) ** 2).mean(dim=(1, 2)))
return torch.stack(errs).mean(0).cpu().numpy()
# ----------------------------------------------------------------------------
# Convenience
# ----------------------------------------------------------------------------
def generate(model, diff, norm, median_age, u15, p65, sex_ratio, year, n=16,
guidance=2.0, n_steps=50, seed=0):
device = next(model.parameters()).device
stats = np.tile(np.array([[median_age, u15, p65, sex_ratio]], dtype=np.float64), (n, 1))
cond = torch.as_tensor(build_cond(stats, year)).to(device)
torch.manual_seed(seed)
x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.0)
return norm.decode(x.cpu().numpy())
def roster_to_pyramid(ages, is_male, weights=None, max_age=100):
"""Household-roster microdata -> (2, 101) shares. Drop missing / 'don't know' ages first."""
ages = np.asarray(ages, dtype=float)
is_male = np.asarray(is_male, dtype=bool)
w = np.ones_like(ages) if weights is None else np.asarray(weights, dtype=float)
ok = np.isfinite(ages) & np.isfinite(w) & (ages >= 0)
ages = np.clip(ages[ok].astype(int), 0, max_age)
is_male, w = is_male[ok], w[ok]
male = np.bincount(ages[is_male], weights=w[is_male], minlength=N_AGES)[:N_AGES]
female = np.bincount(ages[~is_male], weights=w[~is_male], minlength=N_AGES)[:N_AGES]
p = np.stack([male, female])
return p / p.sum()
def plot_repair(obs, rep, truth=None, title=""):
import matplotlib.pyplot as plt
fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True)
ages = np.arange(N_AGES)
for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])):
if truth is not None:
ax.plot(ages, truth[k] * 1e3, lw=1.2, color="0.5", label="truth")
ax.plot(ages, obs[k] * 1e3, lw=0.9, color="tab:red", alpha=0.8, label="observed")
ax.plot(ages, rep[k] * 1e3, lw=1.4, color="tab:blue", label="repaired")
ax.set_title(name)
ax.set_xlabel("age")
axes[0].set_ylabel("share of population (per mille)")
axes[0].legend(frameon=False)
fig.suptitle(title)
fig.tight_layout()
return fig
def plot_generated(samples, title=""):
import matplotlib.pyplot as plt
fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True)
ages = np.arange(N_AGES)
lo, med, hi = np.percentile(samples, [5, 50, 95], axis=0)
for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])):
ax.fill_between(ages, lo[k] * 1e3, hi[k] * 1e3, alpha=0.25, color="tab:blue", label="5-95% of samples")
ax.plot(ages, med[k] * 1e3, color="tab:blue", label="median sample")
ax.set_title(name)
ax.set_xlabel("age")
axes[0].set_ylabel("share of population (per mille)")
axes[0].legend(frameon=False)
fig.suptitle(title)
fig.tight_layout()
return fig
# ----------------------------------------------------------------------------
# Evaluation on held-out countries
# ----------------------------------------------------------------------------
def evaluate_repair(model, diff, norm, shares, years, t_start=200, lam_range=(0.10, 0.35),
seed=0, n_steps=40, n_samples=4,
variants=("obs_post", "obs_each", "pchip_each")):
"""Heap held-out pyramids synthetically, repair, and compare with truth and two classical
baselines (5-point moving average and PCHIP window-total interpolation). MAE is in per-mille of
total population (lower = better). Primary PyramidDiff variant: pchip_each."""
from scipy.ndimage import uniform_filter1d
rng = np.random.default_rng(seed)
truth = _norm_shares(shares)
lam = rng.uniform(*lam_range, size=len(truth))
obs = simulate_heaping(truth, lam, rng)
ma = uniform_filter1d(obs, size=5, axis=-1, mode="nearest")
ma = project_to_windows(ma / ma.sum(axis=(1, 2), keepdims=True), obs)
ma = ma / ma.sum(axis=(1, 2), keepdims=True)
pc = pchip_baseline(obs)
cfg = {"obs_post": ("pyramiddiff_obs_init_posthoc_projection", dict(init="obs", project="post")),
"obs_each": ("pyramiddiff_obs_init_each_step_projection", dict(init="obs", project="each_step")),
"pchip_each": ("pyramiddiff_pchip_init_each_step_projection", dict(init="pchip", project="each_step"))}
preds = {"observed_heaped": obs, "moving_average_5_projected": ma, "pchip_window_totals": pc}
for v in variants:
name, kw = cfg[v]
preds[name] = repair(model, diff, norm, obs, years, t_start=t_start, n_steps=n_steps,
n_samples=n_samples, seed=seed, **kw)
def mae(a):
return float(np.abs(a - truth).mean() * 1e3)
primary = cfg["pchip_each"][0] if "pchip_each" in variants else cfg[variants[0]][0]
return {
"n_pyramids": int(len(truth)),
"t_start": int(t_start),
"primary": primary,
"mae_permille": {k: mae(v) for k, v in preds.items()},
"whipple_mean": {"truth": float(whipple(truth).mean()),
**{k: float(whipple(v).mean()) for k, v in preds.items()}},
"_arrays": {"truth": truth, "obs": obs, "rep": preds[primary], "ma": ma, "pchip": pc},
}
def evaluate_anomaly(model, diff, norm, shares, years, lam_range=(0.10, 0.35), seed=0):
"""AUROC of the denoising-error score for separating clean vs synthetically heaped pyramids."""
from sklearn.metrics import roc_auc_score
device = next(model.parameters()).device
rng = np.random.default_rng(seed)
truth = _norm_shares(shares)
obs = simulate_heaping(truth, rng.uniform(*lam_range, size=len(truth)), rng)
def score(a):
x = torch.as_tensor(norm.encode(a)).to(device)
c = torch.as_tensor(build_cond(summary_stats(a), years)).to(device)
return anomaly_score(model, diff, x, c)
s_clean, s_heap = score(truth), score(obs)
y = np.r_[np.zeros(len(s_clean)), np.ones(len(s_heap))]
return {"auroc_clean_vs_heaped": float(roc_auc_score(y, np.r_[s_clean, s_heap])),
"score_mean_clean": float(s_clean.mean()), "score_mean_heaped": float(s_heap.mean())}
def evaluate_inpaint(model, diff, norm, shares, years, span=(30, 50), seed=0, n_steps=60, guidance=1.5,
batch=512, max_n=None):
"""Hide a contiguous age span (both sexes) of held-out pyramids, fill it with the model (RePaint-style),
and score the hidden ages only (MAE in per mille of total population; lower = better).
Baselines: linear interpolation of the standardised log-share across the gap, and filling with the mean share."""
rng = np.random.default_rng(seed)
truth = _norm_shares(shares)
years = np.broadcast_to(np.asarray(years), (len(truth),))
if max_n is not None and len(truth) > max_n:
keep = np.sort(rng.choice(len(truth), max_n, replace=False))
truth, years = truth[keep], years[keep]
a, b = span
mask = np.ones(truth.shape, dtype=bool)
mask[:, :, a:b] = False
obs = truth.copy()
obs[~mask] = truth[mask].mean()
outs = []
for i in range(0, len(truth), batch):
outs.append(inpaint(model, diff, norm, obs[i:i + batch], mask[i:i + batch], years[i:i + batch],
n_steps=n_steps, guidance=guidance, seed=seed + i))
rep = np.concatenate(outs)
z = norm.encode(truth)
zi = z.copy()
w = (np.arange(a, b) - (a - 1)) / (b - (a - 1))
zi[:, :, a:b] = z[:, :, a - 1:a] * (1 - w) + z[:, :, b:b + 1] * w
lin = norm.decode(zi)
def mae_gap(x):
return float(np.abs(x[:, :, a:b] - truth[:, :, a:b]).mean() * 1e3)
return {"n_pyramids": int(len(truth)), "hidden_ages": [int(a), int(b)],
"mae_permille_hidden_ages": {"pyramiddiff_inpaint": mae_gap(rep),
"linear_interpolation_log_share": mae_gap(lin),
"mean_share_fill": mae_gap(obs)},
"_arrays": {"truth": truth, "rep": rep, "lin": lin}}
# ----------------------------------------------------------------------------
# Save / load
# ----------------------------------------------------------------------------
def save_pretrained(path, model, norm, T, extra=None):
from safetensors.torch import save_file
os.makedirs(path, exist_ok=True)
sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()}
save_file(sd, os.path.join(path, "model.safetensors"))
cfg = {"arch": model.cfg, "T": int(T), "n_ages": N_AGES,
"norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}}
with open(os.path.join(path, "config.json"), "w") as f:
json.dump(cfg, f, indent=2)
def load_pretrained(path, device="cpu"):
from safetensors.torch import load_file
with open(os.path.join(path, "config.json")) as f:
cfg = json.load(f)
model = PyramidUNet(**cfg["arch"])
model.load_state_dict(load_file(os.path.join(path, "model.safetensors")))
model.to(device).eval()
diff = Diffusion(cfg["T"], device)
norm = Normaliser(**cfg["norm"])
return model, diff, norm