""" pyramiddiff.py ============== PyramidDiff: a conditional diffusion model of single-age, sex-specific population structure (age-sex pyramids), trained on UN World Population Prospects (WPP). What it does ------------ 1. generate : sample plausible pyramids given summary conditions (median age, % under 15, % 65+, sex ratio, year) with uncertainty bands. 2. repair : denoise age-heaped / digit-preferenced pyramids (SDEdit-style partial noising + denoising), then project back onto the observed totals of 5-year windows centred on the heaping digits (0 and 5). 3. inpaint : fill in missing ages (RePaint-style). 4. anomaly : score how implausible a pyramid is (denoising error). Representation -------------- A pyramid is a (2, 101) array: [male, female] x ages 0..100 (100 = 100+), expressed as shares of the total population (sums to 1). Shares are log-transformed and standardised before diffusion. """ from __future__ import annotations import copy import json import math import os import numpy as np import pandas as pd import torch import torch.nn as nn import torch.nn.functional as F N_AGES = 101 PAD_LEN = 104 # padded length (divisible by 4) used inside the UNet COND_DIM = 5 # Windows centred on the heaping digits: [0,3) [3,8) [8,13) ... [93,98) [98,101) WINDOW_EDGES = [0] + list(range(3, 99, 5)) + [101] # ---------------------------------------------------------------------------- # Data # ---------------------------------------------------------------------------- def load_wpp(source: str, year_min: int = 1950, year_max: int = 2023): """Load a LOCAL UN WPP CSV (e.g. WPP2024_PopulationBySingleAgeSex_Medium_1950-2023.csv.gz). Return (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands). Column names are matched case-insensitively and the available columns are printed on failure.""" raw = pd.read_csv(source, low_memory=False) low = {c.lower().strip(): c for c in raw.columns} def pick(*names, required=True): for n in names: if n.lower() in low: return low[n.lower()] if required: raise ValueError(f"Could not find any of {names} in the WPP file. Columns found: {list(raw.columns)}") return None c_iso = pick("ISO3_code", "ISO3", required=False) c_loc = pick("Location", "Country", "Area") c_typ = pick("LocTypeName", "LocType", required=False) c_t = pick("Time", "Year") c_age = pick("AgeGrpStart", "Age") c_m, c_f = pick("PopMale", "Male"), pick("PopFemale", "Female") df = raw if c_typ is not None: df = df[df[c_typ].astype(str).str.startswith("Country")] key = c_iso if c_iso is not None else c_loc df = df[df[key].notna() & df[c_t].between(year_min, year_max)] df = df.sort_values([key, c_t, c_age]) size = df.groupby([key, c_t])[c_age].transform("size") df = df[size == N_AGES] if len(df) == 0: raise ValueError("No complete single-age (0..100) blocks found; check you downloaded the " "PopulationBySingleAgeSex file, not the 5-year-group file.") ages = df[c_age].to_numpy().reshape(-1, N_AGES) assert (ages == np.arange(N_AGES)).all(), "Age groups are not 0..100 in every block" counts = ( df[[c_m, c_f]].to_numpy(dtype=np.float64) .reshape(-1, N_AGES, 2).transpose(0, 2, 1) ) meta = df.iloc[::N_AGES][[key, c_loc, c_t]].reset_index(drop=True) meta.columns = ["iso3", "location", "year"] return meta, counts def _read_wpp_xlsx(path, sheet): """Read one UN WPP 'single age' Excel sheet (wide format: one column per age) into a DataFrame with columns iso3, location, year, 0..100. The header row is located automatically.""" xl = pd.ExcelFile(path) names = xl.sheet_names hit = [n for n in names if n.lower().strip() == sheet.lower()] or \ [n for n in names if sheet.lower() in n.lower()] if not hit: raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}") raw = xl.parse(hit[0], header=None) hdr = None for i in range(min(len(raw), 60)): cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()] if "year" in cells and any(c.startswith("iso3") for c in cells): hdr = i break if hdr is None: raise ValueError(f"Could not find the header row (with 'Year' and an ISO3 column) in sheet " f"'{hit[0]}' of {path}.") cols = [str(v).strip() for v in raw.iloc[hdr].tolist()] df = raw.iloc[hdr + 1:].copy() df.columns = cols low = {c.lower(): c for c in cols} c_iso = next(c for c in cols if c.lower().startswith("iso3")) c_year = low["year"] c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso) c_typ = low.get("type") age_cols = {} for c in cols: t = c.replace(".0", "").strip() if t.isdigit() and 0 <= int(t) <= 100: age_cols[int(t)] = c elif t.startswith("100"): age_cols[100] = c missing = [a for a in range(N_AGES) if a not in age_cols] if missing: raise ValueError(f"Age columns {missing[:5]}... not found in sheet '{hit[0]}'; is this the " f"SINGLE AGE file (not 5-year groups)?") if c_typ is not None: df = df[df[c_typ].astype(str).str.startswith("Country")] out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str), "year": pd.to_numeric(df[c_year], errors="coerce")}) ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy() for a in range(N_AGES)}, index=out.index) out = pd.concat([out, ages_df], axis=1) out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna() out["year"] = out["year"].astype(int) return out def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023): """Load the UN WPP Excel files 'Population by Single Age - Male / Female'. Use sheet='Estimates' for 1950-2023 and sheet='Medium variant' for 2024-2100. Returns the same (meta, counts) as load_wpp.""" m = _read_wpp_xlsx(male_path, sheet) f = _read_wpp_xlsx(female_path, sheet) ages = list(range(N_AGES)) m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) common = m.index.intersection(f.index).sort_values() if len(common) == 0: raise ValueError("No overlapping (country, year) rows between the male and female files.") counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1) meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(), "year": [y for _, y in common]}) return meta, counts def counts_to_shares(counts): counts = np.asarray(counts, dtype=np.float64) return counts / counts.sum(axis=(-2, -1), keepdims=True) def split_by_country(meta: pd.DataFrame, val_frac=0.10, test_frac=0.10, seed=0): """Country-level split so that no country appears in more than one split.""" rng = np.random.default_rng(seed) iso = np.array(sorted(meta["iso3"].unique())) rng.shuffle(iso) n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac) test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val] is_test = meta["iso3"].isin(test_iso).to_numpy() is_val = meta["iso3"].isin(val_iso).to_numpy() is_train = ~(is_test | is_val) return is_train, is_val, is_test def summary_stats(shares): """(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females).""" s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES) tot = s.sum(axis=1) tot = tot / tot.sum(axis=-1, keepdims=True) cdf = np.cumsum(tot, axis=-1) idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1) rows = np.arange(len(s)) prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0) frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12) med = idx + np.clip(frac, 0.0, 1.0) u15 = tot[:, :15].sum(-1) p65 = tot[:, 65:].sum(-1) sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12) return np.stack([med, u15, p65, sr], axis=-1) def build_cond(stats, years): """Conditioning vector (n, 5), roughly standardised.""" stats = np.asarray(stats, dtype=np.float64).reshape(-1, 4) med, u15, p65, sr = stats.T years = np.broadcast_to(np.asarray(years, dtype=np.float64), (len(stats),)) c = np.stack( [(med - 30) / 15, (u15 - 0.30) / 0.15, (p65 - 0.10) / 0.10, np.clip((sr - 100) / 15, -3, 3), (years - 1985) / 50], axis=1) return c.astype(np.float32) class Normaliser: """log(share * 2 * N_AGES + eps), standardised with global mean / std.""" def __init__(self, mu=0.0, sigma=1.0, eps=1e-5): self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps) def _log(self, shares): return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps) def fit(self, shares): z = self._log(shares) self.mu, self.sigma = float(z.mean()), float(z.std()) return self def encode(self, shares): return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32) def decode(self, x): x = np.asarray(x, dtype=np.float64) s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES) return s / s.sum(axis=(-2, -1), keepdims=True) # ---------------------------------------------------------------------------- # Model: small conditional 1D UNet with FiLM conditioning and a bottleneck attention # ---------------------------------------------------------------------------- class SinusoidalEmb(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, t): half = self.dim // 2 freqs = torch.exp(-math.log(10000.0) * torch.arange(half, device=t.device) / half) a = t.float()[:, None] * freqs[None] return torch.cat([a.sin(), a.cos()], dim=-1) class Block(nn.Module): def __init__(self, cin, cout, emb_dim): super().__init__() self.n1 = nn.GroupNorm(8, cin) self.c1 = nn.Conv1d(cin, cout, 5, padding=2) self.n2 = nn.GroupNorm(8, cout) self.c2 = nn.Conv1d(cout, cout, 5, padding=2) self.film = nn.Linear(emb_dim, 2 * cout) self.skip = nn.Conv1d(cin, cout, 1) if cin != cout else nn.Identity() def forward(self, x, e): h = self.c1(F.silu(self.n1(x))) scale, shift = self.film(e).chunk(2, dim=1) h = self.n2(h) * (1 + scale[..., None]) + shift[..., None] h = self.c2(F.silu(h)) return h + self.skip(x) class SelfAttn(nn.Module): def __init__(self, c, heads): super().__init__() self.norm = nn.GroupNorm(8, c) self.attn = nn.MultiheadAttention(c, heads, batch_first=True) def forward(self, x): h = self.norm(x).transpose(1, 2) h, _ = self.attn(h, h, h, need_weights=False) return x + h.transpose(1, 2) class PyramidUNet(nn.Module): def __init__(self, base=64, mults=(1, 2, 4), emb_dim=256, cond_dim=COND_DIM, heads=4): super().__init__() self.cfg = dict(base=base, mults=list(mults), emb_dim=emb_dim, cond_dim=cond_dim, heads=heads) self.temb = nn.Sequential(SinusoidalEmb(emb_dim), nn.Linear(emb_dim, emb_dim), nn.SiLU(), nn.Linear(emb_dim, emb_dim)) self.cemb = nn.Sequential(nn.Linear(cond_dim, emb_dim), nn.SiLU(), nn.Linear(emb_dim, emb_dim)) self.null = nn.Parameter(torch.zeros(emb_dim)) # "no condition" embedding (for CFG) self.register_buffer("pos", torch.linspace(-1, 1, PAD_LEN)[None, None, :], persistent=False) self.inp = nn.Conv1d(3, base, 5, padding=2) # male, female, age-position channel chs = [base * m for m in mults] self.down_blocks, self.downs = nn.ModuleList(), nn.ModuleList() c = base for i, ch in enumerate(chs): self.down_blocks.append(nn.ModuleList([Block(c, ch, emb_dim), Block(ch, ch, emb_dim)])) self.downs.append(nn.Conv1d(ch, ch, 3, stride=2, padding=1) if i < len(chs) - 1 else nn.Identity()) c = ch self.mid1, self.attn, self.mid2 = Block(c, c, emb_dim), SelfAttn(c, heads), Block(c, c, emb_dim) self.up_blocks, self.ups = nn.ModuleList(), nn.ModuleList() cur = chs[-1] for i in reversed(range(len(chs))): self.up_blocks.append(nn.ModuleList([Block(cur + chs[i], chs[i], emb_dim), Block(chs[i], chs[i], emb_dim)])) self.ups.append( nn.Sequential(nn.Upsample(scale_factor=2, mode="nearest"), nn.Conv1d(chs[i], chs[i], 3, padding=1)) if i > 0 else nn.Identity()) cur = chs[i] self.out_norm = nn.GroupNorm(8, base) self.out = nn.Conv1d(base, 2, 5, padding=2) def forward(self, x, t, cond=None, drop=None): B = x.shape[0] e = self.temb(t) if cond is None: ce = self.null.expand(B, -1) else: ce = self.cemb(cond) if drop is not None: ce = torch.where(drop[:, None], self.null.expand(B, -1), ce) e = e + ce h = F.pad(x, (0, PAD_LEN - N_AGES), mode="replicate") h = torch.cat([h, self.pos.expand(B, -1, -1)], dim=1) h = self.inp(h) skips = [] for blocks, down in zip(self.down_blocks, self.downs): for b in blocks: h = b(h, e) skips.append(h) h = down(h) h = self.mid2(self.attn(self.mid1(h, e)), e) for blocks, up in zip(self.up_blocks, self.ups): h = torch.cat([h, skips.pop()], dim=1) for b in blocks: h = b(h, e) h = up(h) return self.out(F.silu(self.out_norm(h)))[..., :N_AGES] class EMA: def __init__(self, model, decay=0.999): self.decay = decay self.shadow = copy.deepcopy(model).eval() for p in self.shadow.parameters(): p.requires_grad_(False) @torch.no_grad() def update(self, model, step=None): d = self.decay if step is None else min(self.decay, (1 + step) / (10 + step)) # warm-up for ps, p in zip(self.shadow.parameters(), model.parameters()): ps.mul_(d).add_(p.detach(), alpha=1 - d) # ---------------------------------------------------------------------------- # Diffusion # ---------------------------------------------------------------------------- class Diffusion: def __init__(self, T=500, device="cpu"): self.T = T s = 0.008 t = torch.arange(T + 1, dtype=torch.float64) / T f = torch.cos((t + s) / (1 + s) * math.pi / 2) ** 2 abar = f / f[0] betas = (1 - abar[1:] / abar[:-1]).clamp(1e-5, 0.999) self.abar = torch.cumprod(1 - betas, dim=0).float() self.to(device) def to(self, device): self.abar = self.abar.to(device) return self def q_sample(self, x0, t, noise=None): noise = torch.randn_like(x0) if noise is None else noise a = self.abar[t].view(-1, 1, 1) return a.sqrt() * x0 + (1 - a).sqrt() * noise, noise @torch.no_grad() def sample(model, diff, cond=None, n=None, n_steps=50, guidance=1.5, eta=0.0, x_init=None, t_start=None, known=None, known_mask=None, clip=6.0, x0_hook=None): """DDIM sampler with classifier-free guidance. - plain generation : x_init=None - SDEdit repair : x_init=, t_start= - inpainting : known=, known_mask=<1 where observed> (RePaint-style) """ model.eval() device = next(model.parameters()).device diff.to(device) t_start = diff.T - 1 if t_start is None else int(t_start) if x_init is None: B = n if cond is None else cond.shape[0] if B is None: B = known.shape[0] x = torch.randn(B, 2, N_AGES, device=device) else: B = x_init.shape[0] x = diff.q_sample(x_init, torch.full((B,), t_start, device=device, dtype=torch.long))[0] def predict(xt, t): tt = torch.full((B,), int(t), device=device, dtype=torch.long) if cond is None: return model(xt, tt, None) ec = model(xt, tt, cond) if guidance == 1.0: return ec eu = model(xt, tt, None) return eu + guidance * (ec - eu) ts = [] for t in np.round(np.linspace(t_start, 0, n_steps)).astype(int): if not ts or t != ts[-1]: ts.append(int(t)) for i, t in enumerate(ts): t_prev = ts[i + 1] if i + 1 < len(ts) else -1 eps = predict(x, t) ab = diff.abar[t] ab_prev = diff.abar[t_prev] if t_prev >= 0 else torch.tensor(1.0, device=device) x0 = ((x - (1 - ab).sqrt() * eps) / ab.sqrt()).clamp(-clip, clip) if x0_hook is not None: # e.g. enforce known window totals on the clean estimate x0 = x0_hook(x0) eps = (x - ab.sqrt() * x0) / (1 - ab).sqrt() sigma = eta * ((1 - ab_prev) / (1 - ab)).sqrt() * (1 - ab / ab_prev).clamp(min=0).sqrt() direction = (1 - ab_prev - sigma ** 2).clamp(min=0).sqrt() * eps x = ab_prev.sqrt() * x0 + direction + (sigma * torch.randn_like(x) if eta > 0 else 0.0) if known is not None: if t_prev >= 0: kn = diff.q_sample(known, torch.full((B,), t_prev, device=device, dtype=torch.long))[0] else: kn = known x = known_mask * kn + (1 - known_mask) * x return x # ---------------------------------------------------------------------------- # Training # ---------------------------------------------------------------------------- @torch.no_grad() def val_loss(model, diff, X, C, seed=123): g = torch.Generator(device=X.device).manual_seed(seed) losses = [] for t_val in (25, 75, 150, 250, 400): t = torch.full((X.shape[0],), t_val, device=X.device, dtype=torch.long) noise = torch.randn(X.shape, generator=g, device=X.device) xt, _ = diff.q_sample(X, t, noise) losses.append(F.mse_loss(model(xt, t, C), noise).item()) return float(np.mean(losses)) def train(model, diff, x, c, x_val=None, c_val=None, steps=12000, bs=256, lr=3e-4, p_uncond=0.1, ema_decay=0.999, log_every=500, device=None, seed=0): device = device or ("cuda" if torch.cuda.is_available() else "cpu") torch.manual_seed(seed) model.to(device).train() diff.to(device) ema = EMA(model, ema_decay) ema.shadow.to(device) X, C = torch.as_tensor(x).to(device), torch.as_tensor(c).to(device) Xv = torch.as_tensor(x_val).to(device) if x_val is not None else None Cv = torch.as_tensor(c_val).to(device) if c_val is not None else None opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4) history, run = [], [] for step in range(steps): lr_now = lr * min(1.0, (step + 1) / 500) * 0.5 * (1 + math.cos(math.pi * step / steps)) for g in opt.param_groups: g["lr"] = lr_now idx = torch.randint(0, X.shape[0], (bs,), device=device) t = torch.randint(0, diff.T, (bs,), device=device) xt, noise = diff.q_sample(X[idx], t) drop = torch.rand(bs, device=device) < p_uncond loss = F.mse_loss(model(xt, t, C[idx], drop), noise) opt.zero_grad(set_to_none=True) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() ema.update(model, step) run.append(loss.item()) if (step + 1) % log_every == 0 or step + 1 == steps: rec = {"step": step + 1, "train_loss": float(np.mean(run))} run = [] if Xv is not None: rec["val_loss_ema"] = val_loss(ema.shadow, diff, Xv, Cv) history.append(rec) print(rec) model.eval() return ema, history # ---------------------------------------------------------------------------- # Age heaping: simulator, index, repair # ---------------------------------------------------------------------------- def simulate_heaping(shares, lam, rng=None, digit0_boost=1.6, noise_sd=0.03): """Mimic digit preference. A fraction `lam` (per pyramid) of people aged within +-2 of a multiple of 5 report that multiple instead (stronger for multiples of 10), plus mild multiplicative roster noise. shares: (n, 2, 101); lam: scalar or (n,).""" rng = np.random.default_rng() if rng is None else rng out = np.array(shares, dtype=np.float64).reshape(-1, 2, N_AGES) lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (len(out),)) for h in range(5, 100, 5): boost = digit0_boost if h % 10 == 0 else 1.0 for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)): frac = np.clip(lam * boost * w, 0, 0.9)[:, None] moved = out[:, :, h + d] * frac out[:, :, h + d] -= moved out[:, :, h] += moved out *= np.exp(rng.normal(0.0, noise_sd, out.shape)) return out / out.sum(axis=(1, 2), keepdims=True) def whipple(shares): """Whipple index (both sexes), ages 23-62. ~100 = no preference, 500 = only 0/5.""" tot = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES).sum(axis=1) ages = np.arange(23, 63) num = tot[:, ages[ages % 5 == 0]].sum(-1) den = tot[:, 23:63].sum(-1) return 500.0 * num / den def project_to_windows(rep, obs): """Rescale `rep` so that totals of windows centred on heaping digits match `obs` (heaping moves people within +-2 years, so these totals are approximately conserved).""" rep = np.asarray(rep, dtype=np.float64) obs = np.asarray(obs, dtype=np.float64) out = rep.copy() for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]): r = rep[..., a:b].sum(-1, keepdims=True) o = obs[..., a:b].sum(-1, keepdims=True) out[..., a:b] = rep[..., a:b] * (o / np.maximum(r, 1e-12)) return out def _norm_shares(a): a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES) return a / a.sum(axis=(1, 2), keepdims=True) def _decode_t(x, norm): s = (torch.exp(x * norm.sigma + norm.mu) - norm.eps).clamp(min=0.0) / (2 * N_AGES) return s / s.sum(dim=(-2, -1), keepdim=True) def _encode_t(s, norm): return (torch.log(s * 2 * N_AGES + norm.eps) - norm.mu) / norm.sigma def _project_t(rep, obs): out = rep.clone() for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]): r = rep[..., a:b].sum(-1, keepdim=True) o = obs[..., a:b].sum(-1, keepdim=True) out[..., a:b] = rep[..., a:b] * (o / r.clamp(min=1e-12)) return out def pchip_baseline(obs): """Classical window-total-preserving smoother: monotone (PCHIP) interpolation of the cumulative population at the window edges, differenced back to single ages. Heaping cannot move the window totals much, so this removes digit preference without a learned model.""" from scipy.interpolate import PchipInterpolator obs = _norm_shares(obs) edges = np.array(WINDOW_EDGES) tot = np.add.reduceat(obs, edges[:-1], axis=-1) # (n, 2, n_windows) cum = np.concatenate([np.zeros(tot.shape[:-1] + (1,)), np.cumsum(tot, axis=-1)], axis=-1) f = PchipInterpolator(edges, cum, axis=-1) out = np.clip(np.diff(f(np.arange(N_AGES + 1)), axis=-1), 0.0, None) return out / out.sum(axis=(1, 2), keepdims=True) def repair(model, diff, norm, obs_shares, years, t_start=200, n_steps=40, n_samples=4, use_cond=True, project="each_step", init="pchip", seed=0): """Repair heaped pyramids. Returns shares (n, 2, 101). init : "pchip" starts from the classical window-total-preserving smoothing (in-distribution input for the model); "obs" starts from the raw heaped pyramid. project : "each_step" re-imposes the observed window totals on the clean estimate at every denoising step (the model only decides the within-window shape); "post" projects once at the end; "none" does not project. """ device = next(model.parameters()).device obs = _norm_shares(obs_shares) start = pchip_baseline(obs) if init == "pchip" else obs x_start = torch.as_tensor(norm.encode(start)).to(device) cond = None if use_cond: cond = torch.as_tensor(build_cond(summary_stats(obs), years)).to(device) hook = None if project == "each_step": obs_t = torch.as_tensor(obs, dtype=torch.float32, device=device) def hook(x0): return _encode_t(_project_t(_decode_t(x0, norm), obs_t), norm).clamp(-6.0, 6.0) torch.manual_seed(seed) outs = [] for _ in range(n_samples): x = sample(model, diff, cond, n_steps=n_steps, guidance=1.0, eta=0.0, x_init=x_start, t_start=t_start, x0_hook=hook) outs.append(norm.decode(x.cpu().numpy())) rep = np.mean(outs, axis=0) if project in ("each_step", "post"): rep = project_to_windows(rep, obs) return rep / rep.sum(axis=(1, 2), keepdims=True) def inpaint(model, diff, norm, obs_shares, observed_mask, years, n_steps=60, guidance=1.5, seed=0): """Fill unobserved ages. observed_mask: (2,101) or (n,2,101) bool, True = observed. Values at unobserved positions in obs_shares are ignored (use any positive number).""" device = next(model.parameters()).device obs = np.clip(_norm_shares(obs_shares), 1e-9, None) mask = np.broadcast_to(np.asarray(observed_mask, dtype=bool), obs.shape) known = torch.as_tensor(norm.encode(obs)).to(device) m = torch.as_tensor(mask.astype(np.float32)).to(device) # conditioning from the observed part only: fill the gaps with the mean observed share filled = np.where(mask, obs, obs[mask].mean()) cond = torch.as_tensor(build_cond(summary_stats(filled), years)).to(device) torch.manual_seed(seed) x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.3, known=known, known_mask=m) return norm.decode(x.cpu().numpy()) @torch.no_grad() def anomaly_score(model, diff, x, cond, ts=(20, 40, 60, 80, 120), seed=0): """Mean noise-prediction error at low noise levels; higher = less plausible pyramid.""" model.eval() diff.to(x.device) g = torch.Generator(device=x.device).manual_seed(seed) errs = [] for t_val in ts: t = torch.full((x.shape[0],), t_val, device=x.device, dtype=torch.long) noise = torch.randn(x.shape, generator=g, device=x.device) xt, _ = diff.q_sample(x, t, noise) errs.append(((model(xt, t, cond) - noise) ** 2).mean(dim=(1, 2))) return torch.stack(errs).mean(0).cpu().numpy() # ---------------------------------------------------------------------------- # Convenience # ---------------------------------------------------------------------------- def generate(model, diff, norm, median_age, u15, p65, sex_ratio, year, n=16, guidance=2.0, n_steps=50, seed=0): device = next(model.parameters()).device stats = np.tile(np.array([[median_age, u15, p65, sex_ratio]], dtype=np.float64), (n, 1)) cond = torch.as_tensor(build_cond(stats, year)).to(device) torch.manual_seed(seed) x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.0) return norm.decode(x.cpu().numpy()) def roster_to_pyramid(ages, is_male, weights=None, max_age=100): """Household-roster microdata -> (2, 101) shares. Drop missing / 'don't know' ages first.""" ages = np.asarray(ages, dtype=float) is_male = np.asarray(is_male, dtype=bool) w = np.ones_like(ages) if weights is None else np.asarray(weights, dtype=float) ok = np.isfinite(ages) & np.isfinite(w) & (ages >= 0) ages = np.clip(ages[ok].astype(int), 0, max_age) is_male, w = is_male[ok], w[ok] male = np.bincount(ages[is_male], weights=w[is_male], minlength=N_AGES)[:N_AGES] female = np.bincount(ages[~is_male], weights=w[~is_male], minlength=N_AGES)[:N_AGES] p = np.stack([male, female]) return p / p.sum() def plot_repair(obs, rep, truth=None, title=""): import matplotlib.pyplot as plt fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True) ages = np.arange(N_AGES) for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])): if truth is not None: ax.plot(ages, truth[k] * 1e3, lw=1.2, color="0.5", label="truth") ax.plot(ages, obs[k] * 1e3, lw=0.9, color="tab:red", alpha=0.8, label="observed") ax.plot(ages, rep[k] * 1e3, lw=1.4, color="tab:blue", label="repaired") ax.set_title(name) ax.set_xlabel("age") axes[0].set_ylabel("share of population (per mille)") axes[0].legend(frameon=False) fig.suptitle(title) fig.tight_layout() return fig def plot_generated(samples, title=""): import matplotlib.pyplot as plt fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True) ages = np.arange(N_AGES) lo, med, hi = np.percentile(samples, [5, 50, 95], axis=0) for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])): ax.fill_between(ages, lo[k] * 1e3, hi[k] * 1e3, alpha=0.25, color="tab:blue", label="5-95% of samples") ax.plot(ages, med[k] * 1e3, color="tab:blue", label="median sample") ax.set_title(name) ax.set_xlabel("age") axes[0].set_ylabel("share of population (per mille)") axes[0].legend(frameon=False) fig.suptitle(title) fig.tight_layout() return fig # ---------------------------------------------------------------------------- # Evaluation on held-out countries # ---------------------------------------------------------------------------- def evaluate_repair(model, diff, norm, shares, years, t_start=200, lam_range=(0.10, 0.35), seed=0, n_steps=40, n_samples=4, variants=("obs_post", "obs_each", "pchip_each")): """Heap held-out pyramids synthetically, repair, and compare with truth and two classical baselines (5-point moving average and PCHIP window-total interpolation). MAE is in per-mille of total population (lower = better). Primary PyramidDiff variant: pchip_each.""" from scipy.ndimage import uniform_filter1d rng = np.random.default_rng(seed) truth = _norm_shares(shares) lam = rng.uniform(*lam_range, size=len(truth)) obs = simulate_heaping(truth, lam, rng) ma = uniform_filter1d(obs, size=5, axis=-1, mode="nearest") ma = project_to_windows(ma / ma.sum(axis=(1, 2), keepdims=True), obs) ma = ma / ma.sum(axis=(1, 2), keepdims=True) pc = pchip_baseline(obs) cfg = {"obs_post": ("pyramiddiff_obs_init_posthoc_projection", dict(init="obs", project="post")), "obs_each": ("pyramiddiff_obs_init_each_step_projection", dict(init="obs", project="each_step")), "pchip_each": ("pyramiddiff_pchip_init_each_step_projection", dict(init="pchip", project="each_step"))} preds = {"observed_heaped": obs, "moving_average_5_projected": ma, "pchip_window_totals": pc} for v in variants: name, kw = cfg[v] preds[name] = repair(model, diff, norm, obs, years, t_start=t_start, n_steps=n_steps, n_samples=n_samples, seed=seed, **kw) def mae(a): return float(np.abs(a - truth).mean() * 1e3) primary = cfg["pchip_each"][0] if "pchip_each" in variants else cfg[variants[0]][0] return { "n_pyramids": int(len(truth)), "t_start": int(t_start), "primary": primary, "mae_permille": {k: mae(v) for k, v in preds.items()}, "whipple_mean": {"truth": float(whipple(truth).mean()), **{k: float(whipple(v).mean()) for k, v in preds.items()}}, "_arrays": {"truth": truth, "obs": obs, "rep": preds[primary], "ma": ma, "pchip": pc}, } def evaluate_anomaly(model, diff, norm, shares, years, lam_range=(0.10, 0.35), seed=0): """AUROC of the denoising-error score for separating clean vs synthetically heaped pyramids.""" from sklearn.metrics import roc_auc_score device = next(model.parameters()).device rng = np.random.default_rng(seed) truth = _norm_shares(shares) obs = simulate_heaping(truth, rng.uniform(*lam_range, size=len(truth)), rng) def score(a): x = torch.as_tensor(norm.encode(a)).to(device) c = torch.as_tensor(build_cond(summary_stats(a), years)).to(device) return anomaly_score(model, diff, x, c) s_clean, s_heap = score(truth), score(obs) y = np.r_[np.zeros(len(s_clean)), np.ones(len(s_heap))] return {"auroc_clean_vs_heaped": float(roc_auc_score(y, np.r_[s_clean, s_heap])), "score_mean_clean": float(s_clean.mean()), "score_mean_heaped": float(s_heap.mean())} def evaluate_inpaint(model, diff, norm, shares, years, span=(30, 50), seed=0, n_steps=60, guidance=1.5, batch=512, max_n=None): """Hide a contiguous age span (both sexes) of held-out pyramids, fill it with the model (RePaint-style), and score the hidden ages only (MAE in per mille of total population; lower = better). Baselines: linear interpolation of the standardised log-share across the gap, and filling with the mean share.""" rng = np.random.default_rng(seed) truth = _norm_shares(shares) years = np.broadcast_to(np.asarray(years), (len(truth),)) if max_n is not None and len(truth) > max_n: keep = np.sort(rng.choice(len(truth), max_n, replace=False)) truth, years = truth[keep], years[keep] a, b = span mask = np.ones(truth.shape, dtype=bool) mask[:, :, a:b] = False obs = truth.copy() obs[~mask] = truth[mask].mean() outs = [] for i in range(0, len(truth), batch): outs.append(inpaint(model, diff, norm, obs[i:i + batch], mask[i:i + batch], years[i:i + batch], n_steps=n_steps, guidance=guidance, seed=seed + i)) rep = np.concatenate(outs) z = norm.encode(truth) zi = z.copy() w = (np.arange(a, b) - (a - 1)) / (b - (a - 1)) zi[:, :, a:b] = z[:, :, a - 1:a] * (1 - w) + z[:, :, b:b + 1] * w lin = norm.decode(zi) def mae_gap(x): return float(np.abs(x[:, :, a:b] - truth[:, :, a:b]).mean() * 1e3) return {"n_pyramids": int(len(truth)), "hidden_ages": [int(a), int(b)], "mae_permille_hidden_ages": {"pyramiddiff_inpaint": mae_gap(rep), "linear_interpolation_log_share": mae_gap(lin), "mean_share_fill": mae_gap(obs)}, "_arrays": {"truth": truth, "rep": rep, "lin": lin}} # ---------------------------------------------------------------------------- # Save / load # ---------------------------------------------------------------------------- def save_pretrained(path, model, norm, T, extra=None): from safetensors.torch import save_file os.makedirs(path, exist_ok=True) sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()} save_file(sd, os.path.join(path, "model.safetensors")) cfg = {"arch": model.cfg, "T": int(T), "n_ages": N_AGES, "norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}} with open(os.path.join(path, "config.json"), "w") as f: json.dump(cfg, f, indent=2) def load_pretrained(path, device="cpu"): from safetensors.torch import load_file with open(os.path.join(path, "config.json")) as f: cfg = json.load(f) model = PyramidUNet(**cfg["arch"]) model.load_state_dict(load_file(os.path.join(path, "model.safetensors"))) model.to(device).eval() diff = Diffusion(cfg["T"], device) norm = Normaliser(**cfg["norm"]) return model, diff, norm