Download pyramiddiff.py from SallySims/pyramiddiff-wpp2024: direct link, hf CLI and curl.
- Browser
- Download file 36.3 kB
-
https://huggingface.co/SallySims/pyramiddiff-wpp2024/resolve/main/pyramiddiff.py
- Command line
-
hf download hf://SallySims/pyramiddiff-wpp2024/pyramiddiff.py
-
curl -L -o pyramiddiff.py https://huggingface.co/SallySims/pyramiddiff-wpp2024/resolve/main/pyramiddiff.py
36.3 kB
| """ | |
| pyramiddiff.py | |
| ============== | |
| PyramidDiff: a conditional diffusion model of single-age, sex-specific population | |
| structure (age-sex pyramids), trained on UN World Population Prospects (WPP). | |
| What it does | |
| ------------ | |
| 1. generate : sample plausible pyramids given summary conditions | |
| (median age, % under 15, % 65+, sex ratio, year) with uncertainty bands. | |
| 2. repair : denoise age-heaped / digit-preferenced pyramids (SDEdit-style partial | |
| noising + denoising), then project back onto the observed totals of | |
| 5-year windows centred on the heaping digits (0 and 5). | |
| 3. inpaint : fill in missing ages (RePaint-style). | |
| 4. anomaly : score how implausible a pyramid is (denoising error). | |
| Representation | |
| -------------- | |
| A pyramid is a (2, 101) array: [male, female] x ages 0..100 (100 = 100+), expressed | |
| as shares of the total population (sums to 1). Shares are log-transformed and | |
| standardised before diffusion. | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import json | |
| import math | |
| import os | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| N_AGES = 101 | |
| PAD_LEN = 104 # padded length (divisible by 4) used inside the UNet | |
| COND_DIM = 5 | |
| # Windows centred on the heaping digits: [0,3) [3,8) [8,13) ... [93,98) [98,101) | |
| WINDOW_EDGES = [0] + list(range(3, 99, 5)) + [101] | |
| # ---------------------------------------------------------------------------- | |
| # Data | |
| # ---------------------------------------------------------------------------- | |
| def load_wpp(source: str, year_min: int = 1950, year_max: int = 2023): | |
| """Load a LOCAL UN WPP CSV (e.g. WPP2024_PopulationBySingleAgeSex_Medium_1950-2023.csv.gz). | |
| Return (meta DataFrame[iso3, location, year], counts array (n, 2, 101) in thousands). | |
| Column names are matched case-insensitively and the available columns are printed on failure.""" | |
| raw = pd.read_csv(source, low_memory=False) | |
| low = {c.lower().strip(): c for c in raw.columns} | |
| def pick(*names, required=True): | |
| for n in names: | |
| if n.lower() in low: | |
| return low[n.lower()] | |
| if required: | |
| raise ValueError(f"Could not find any of {names} in the WPP file. Columns found: {list(raw.columns)}") | |
| return None | |
| c_iso = pick("ISO3_code", "ISO3", required=False) | |
| c_loc = pick("Location", "Country", "Area") | |
| c_typ = pick("LocTypeName", "LocType", required=False) | |
| c_t = pick("Time", "Year") | |
| c_age = pick("AgeGrpStart", "Age") | |
| c_m, c_f = pick("PopMale", "Male"), pick("PopFemale", "Female") | |
| df = raw | |
| if c_typ is not None: | |
| df = df[df[c_typ].astype(str).str.startswith("Country")] | |
| key = c_iso if c_iso is not None else c_loc | |
| df = df[df[key].notna() & df[c_t].between(year_min, year_max)] | |
| df = df.sort_values([key, c_t, c_age]) | |
| size = df.groupby([key, c_t])[c_age].transform("size") | |
| df = df[size == N_AGES] | |
| if len(df) == 0: | |
| raise ValueError("No complete single-age (0..100) blocks found; check you downloaded the " | |
| "PopulationBySingleAgeSex file, not the 5-year-group file.") | |
| ages = df[c_age].to_numpy().reshape(-1, N_AGES) | |
| assert (ages == np.arange(N_AGES)).all(), "Age groups are not 0..100 in every block" | |
| counts = ( | |
| df[[c_m, c_f]].to_numpy(dtype=np.float64) | |
| .reshape(-1, N_AGES, 2).transpose(0, 2, 1) | |
| ) | |
| meta = df.iloc[::N_AGES][[key, c_loc, c_t]].reset_index(drop=True) | |
| meta.columns = ["iso3", "location", "year"] | |
| return meta, counts | |
| def _read_wpp_xlsx(path, sheet): | |
| """Read one UN WPP 'single age' Excel sheet (wide format: one column per age) into a DataFrame | |
| with columns iso3, location, year, 0..100. The header row is located automatically.""" | |
| xl = pd.ExcelFile(path) | |
| names = xl.sheet_names | |
| hit = [n for n in names if n.lower().strip() == sheet.lower()] or \ | |
| [n for n in names if sheet.lower() in n.lower()] | |
| if not hit: | |
| raise ValueError(f"No sheet like '{sheet}' in {path}. Sheets found: {names}") | |
| raw = xl.parse(hit[0], header=None) | |
| hdr = None | |
| for i in range(min(len(raw), 60)): | |
| cells = [str(v).strip().lower() for v in raw.iloc[i].tolist()] | |
| if "year" in cells and any(c.startswith("iso3") for c in cells): | |
| hdr = i | |
| break | |
| if hdr is None: | |
| raise ValueError(f"Could not find the header row (with 'Year' and an ISO3 column) in sheet " | |
| f"'{hit[0]}' of {path}.") | |
| cols = [str(v).strip() for v in raw.iloc[hdr].tolist()] | |
| df = raw.iloc[hdr + 1:].copy() | |
| df.columns = cols | |
| low = {c.lower(): c for c in cols} | |
| c_iso = next(c for c in cols if c.lower().startswith("iso3")) | |
| c_year = low["year"] | |
| c_loc = next((c for c in cols if c.lower().startswith("region") or "country or area" in c.lower()), c_iso) | |
| c_typ = low.get("type") | |
| age_cols = {} | |
| for c in cols: | |
| t = c.replace(".0", "").strip() | |
| if t.isdigit() and 0 <= int(t) <= 100: | |
| age_cols[int(t)] = c | |
| elif t.startswith("100"): | |
| age_cols[100] = c | |
| missing = [a for a in range(N_AGES) if a not in age_cols] | |
| if missing: | |
| raise ValueError(f"Age columns {missing[:5]}... not found in sheet '{hit[0]}'; is this the " | |
| f"SINGLE AGE file (not 5-year groups)?") | |
| if c_typ is not None: | |
| df = df[df[c_typ].astype(str).str.startswith("Country")] | |
| out = pd.DataFrame({"iso3": df[c_iso].astype(str).str.strip(), "location": df[c_loc].astype(str), | |
| "year": pd.to_numeric(df[c_year], errors="coerce")}) | |
| ages_df = pd.DataFrame({a: pd.to_numeric(df[age_cols[a]], errors="coerce").to_numpy() | |
| for a in range(N_AGES)}, index=out.index) | |
| out = pd.concat([out, ages_df], axis=1) | |
| out = out[out["year"].notna() & out["iso3"].str.len().eq(3)].dropna() | |
| out["year"] = out["year"].astype(int) | |
| return out | |
| def load_wpp_xlsx(male_path, female_path, sheet="Estimates", year_min=1950, year_max=2023): | |
| """Load the UN WPP Excel files 'Population by Single Age - Male / Female'. | |
| Use sheet='Estimates' for 1950-2023 and sheet='Medium variant' for 2024-2100. | |
| Returns the same (meta, counts) as load_wpp.""" | |
| m = _read_wpp_xlsx(male_path, sheet) | |
| f = _read_wpp_xlsx(female_path, sheet) | |
| ages = list(range(N_AGES)) | |
| m = m[m["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) | |
| f = f[f["year"].between(year_min, year_max)].drop_duplicates(["iso3", "year"]).set_index(["iso3", "year"]) | |
| common = m.index.intersection(f.index).sort_values() | |
| if len(common) == 0: | |
| raise ValueError("No overlapping (country, year) rows between the male and female files.") | |
| counts = np.stack([m.loc[common, ages].to_numpy(float), f.loc[common, ages].to_numpy(float)], axis=1) | |
| meta = pd.DataFrame({"iso3": [i for i, _ in common], "location": m.loc[common, "location"].to_numpy(), | |
| "year": [y for _, y in common]}) | |
| return meta, counts | |
| def counts_to_shares(counts): | |
| counts = np.asarray(counts, dtype=np.float64) | |
| return counts / counts.sum(axis=(-2, -1), keepdims=True) | |
| def split_by_country(meta: pd.DataFrame, val_frac=0.10, test_frac=0.10, seed=0): | |
| """Country-level split so that no country appears in more than one split.""" | |
| rng = np.random.default_rng(seed) | |
| iso = np.array(sorted(meta["iso3"].unique())) | |
| rng.shuffle(iso) | |
| n_val, n_test = int(len(iso) * val_frac), int(len(iso) * test_frac) | |
| test_iso, val_iso = iso[:n_test], iso[n_test:n_test + n_val] | |
| is_test = meta["iso3"].isin(test_iso).to_numpy() | |
| is_val = meta["iso3"].isin(val_iso).to_numpy() | |
| is_train = ~(is_test | is_val) | |
| return is_train, is_val, is_test | |
| def summary_stats(shares): | |
| """(n, 4): median age, share <15, share 65+, sex ratio (males per 100 females).""" | |
| s = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES) | |
| tot = s.sum(axis=1) | |
| tot = tot / tot.sum(axis=-1, keepdims=True) | |
| cdf = np.cumsum(tot, axis=-1) | |
| idx = np.minimum((cdf < 0.5).sum(axis=-1), N_AGES - 1) | |
| rows = np.arange(len(s)) | |
| prev = np.where(idx > 0, cdf[rows, np.maximum(idx - 1, 0)], 0.0) | |
| frac = (0.5 - prev) / np.maximum(tot[rows, idx], 1e-12) | |
| med = idx + np.clip(frac, 0.0, 1.0) | |
| u15 = tot[:, :15].sum(-1) | |
| p65 = tot[:, 65:].sum(-1) | |
| sr = 100.0 * s[:, 0].sum(-1) / np.maximum(s[:, 1].sum(-1), 1e-12) | |
| return np.stack([med, u15, p65, sr], axis=-1) | |
| def build_cond(stats, years): | |
| """Conditioning vector (n, 5), roughly standardised.""" | |
| stats = np.asarray(stats, dtype=np.float64).reshape(-1, 4) | |
| med, u15, p65, sr = stats.T | |
| years = np.broadcast_to(np.asarray(years, dtype=np.float64), (len(stats),)) | |
| c = np.stack( | |
| [(med - 30) / 15, (u15 - 0.30) / 0.15, (p65 - 0.10) / 0.10, | |
| np.clip((sr - 100) / 15, -3, 3), (years - 1985) / 50], axis=1) | |
| return c.astype(np.float32) | |
| class Normaliser: | |
| """log(share * 2 * N_AGES + eps), standardised with global mean / std.""" | |
| def __init__(self, mu=0.0, sigma=1.0, eps=1e-5): | |
| self.mu, self.sigma, self.eps = float(mu), float(sigma), float(eps) | |
| def _log(self, shares): | |
| return np.log(np.asarray(shares, dtype=np.float64) * 2 * N_AGES + self.eps) | |
| def fit(self, shares): | |
| z = self._log(shares) | |
| self.mu, self.sigma = float(z.mean()), float(z.std()) | |
| return self | |
| def encode(self, shares): | |
| return ((self._log(shares) - self.mu) / self.sigma).astype(np.float32) | |
| def decode(self, x): | |
| x = np.asarray(x, dtype=np.float64) | |
| s = np.clip(np.exp(x * self.sigma + self.mu) - self.eps, 0.0, None) / (2 * N_AGES) | |
| return s / s.sum(axis=(-2, -1), keepdims=True) | |
| # ---------------------------------------------------------------------------- | |
| # Model: small conditional 1D UNet with FiLM conditioning and a bottleneck attention | |
| # ---------------------------------------------------------------------------- | |
| class SinusoidalEmb(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| def forward(self, t): | |
| half = self.dim // 2 | |
| freqs = torch.exp(-math.log(10000.0) * torch.arange(half, device=t.device) / half) | |
| a = t.float()[:, None] * freqs[None] | |
| return torch.cat([a.sin(), a.cos()], dim=-1) | |
| class Block(nn.Module): | |
| def __init__(self, cin, cout, emb_dim): | |
| super().__init__() | |
| self.n1 = nn.GroupNorm(8, cin) | |
| self.c1 = nn.Conv1d(cin, cout, 5, padding=2) | |
| self.n2 = nn.GroupNorm(8, cout) | |
| self.c2 = nn.Conv1d(cout, cout, 5, padding=2) | |
| self.film = nn.Linear(emb_dim, 2 * cout) | |
| self.skip = nn.Conv1d(cin, cout, 1) if cin != cout else nn.Identity() | |
| def forward(self, x, e): | |
| h = self.c1(F.silu(self.n1(x))) | |
| scale, shift = self.film(e).chunk(2, dim=1) | |
| h = self.n2(h) * (1 + scale[..., None]) + shift[..., None] | |
| h = self.c2(F.silu(h)) | |
| return h + self.skip(x) | |
| class SelfAttn(nn.Module): | |
| def __init__(self, c, heads): | |
| super().__init__() | |
| self.norm = nn.GroupNorm(8, c) | |
| self.attn = nn.MultiheadAttention(c, heads, batch_first=True) | |
| def forward(self, x): | |
| h = self.norm(x).transpose(1, 2) | |
| h, _ = self.attn(h, h, h, need_weights=False) | |
| return x + h.transpose(1, 2) | |
| class PyramidUNet(nn.Module): | |
| def __init__(self, base=64, mults=(1, 2, 4), emb_dim=256, cond_dim=COND_DIM, heads=4): | |
| super().__init__() | |
| self.cfg = dict(base=base, mults=list(mults), emb_dim=emb_dim, cond_dim=cond_dim, heads=heads) | |
| self.temb = nn.Sequential(SinusoidalEmb(emb_dim), nn.Linear(emb_dim, emb_dim), | |
| nn.SiLU(), nn.Linear(emb_dim, emb_dim)) | |
| self.cemb = nn.Sequential(nn.Linear(cond_dim, emb_dim), nn.SiLU(), nn.Linear(emb_dim, emb_dim)) | |
| self.null = nn.Parameter(torch.zeros(emb_dim)) # "no condition" embedding (for CFG) | |
| self.register_buffer("pos", torch.linspace(-1, 1, PAD_LEN)[None, None, :], persistent=False) | |
| self.inp = nn.Conv1d(3, base, 5, padding=2) # male, female, age-position channel | |
| chs = [base * m for m in mults] | |
| self.down_blocks, self.downs = nn.ModuleList(), nn.ModuleList() | |
| c = base | |
| for i, ch in enumerate(chs): | |
| self.down_blocks.append(nn.ModuleList([Block(c, ch, emb_dim), Block(ch, ch, emb_dim)])) | |
| self.downs.append(nn.Conv1d(ch, ch, 3, stride=2, padding=1) if i < len(chs) - 1 else nn.Identity()) | |
| c = ch | |
| self.mid1, self.attn, self.mid2 = Block(c, c, emb_dim), SelfAttn(c, heads), Block(c, c, emb_dim) | |
| self.up_blocks, self.ups = nn.ModuleList(), nn.ModuleList() | |
| cur = chs[-1] | |
| for i in reversed(range(len(chs))): | |
| self.up_blocks.append(nn.ModuleList([Block(cur + chs[i], chs[i], emb_dim), | |
| Block(chs[i], chs[i], emb_dim)])) | |
| self.ups.append( | |
| nn.Sequential(nn.Upsample(scale_factor=2, mode="nearest"), | |
| nn.Conv1d(chs[i], chs[i], 3, padding=1)) if i > 0 else nn.Identity()) | |
| cur = chs[i] | |
| self.out_norm = nn.GroupNorm(8, base) | |
| self.out = nn.Conv1d(base, 2, 5, padding=2) | |
| def forward(self, x, t, cond=None, drop=None): | |
| B = x.shape[0] | |
| e = self.temb(t) | |
| if cond is None: | |
| ce = self.null.expand(B, -1) | |
| else: | |
| ce = self.cemb(cond) | |
| if drop is not None: | |
| ce = torch.where(drop[:, None], self.null.expand(B, -1), ce) | |
| e = e + ce | |
| h = F.pad(x, (0, PAD_LEN - N_AGES), mode="replicate") | |
| h = torch.cat([h, self.pos.expand(B, -1, -1)], dim=1) | |
| h = self.inp(h) | |
| skips = [] | |
| for blocks, down in zip(self.down_blocks, self.downs): | |
| for b in blocks: | |
| h = b(h, e) | |
| skips.append(h) | |
| h = down(h) | |
| h = self.mid2(self.attn(self.mid1(h, e)), e) | |
| for blocks, up in zip(self.up_blocks, self.ups): | |
| h = torch.cat([h, skips.pop()], dim=1) | |
| for b in blocks: | |
| h = b(h, e) | |
| h = up(h) | |
| return self.out(F.silu(self.out_norm(h)))[..., :N_AGES] | |
| class EMA: | |
| def __init__(self, model, decay=0.999): | |
| self.decay = decay | |
| self.shadow = copy.deepcopy(model).eval() | |
| for p in self.shadow.parameters(): | |
| p.requires_grad_(False) | |
| def update(self, model, step=None): | |
| d = self.decay if step is None else min(self.decay, (1 + step) / (10 + step)) # warm-up | |
| for ps, p in zip(self.shadow.parameters(), model.parameters()): | |
| ps.mul_(d).add_(p.detach(), alpha=1 - d) | |
| # ---------------------------------------------------------------------------- | |
| # Diffusion | |
| # ---------------------------------------------------------------------------- | |
| class Diffusion: | |
| def __init__(self, T=500, device="cpu"): | |
| self.T = T | |
| s = 0.008 | |
| t = torch.arange(T + 1, dtype=torch.float64) / T | |
| f = torch.cos((t + s) / (1 + s) * math.pi / 2) ** 2 | |
| abar = f / f[0] | |
| betas = (1 - abar[1:] / abar[:-1]).clamp(1e-5, 0.999) | |
| self.abar = torch.cumprod(1 - betas, dim=0).float() | |
| self.to(device) | |
| def to(self, device): | |
| self.abar = self.abar.to(device) | |
| return self | |
| def q_sample(self, x0, t, noise=None): | |
| noise = torch.randn_like(x0) if noise is None else noise | |
| a = self.abar[t].view(-1, 1, 1) | |
| return a.sqrt() * x0 + (1 - a).sqrt() * noise, noise | |
| def sample(model, diff, cond=None, n=None, n_steps=50, guidance=1.5, eta=0.0, | |
| x_init=None, t_start=None, known=None, known_mask=None, clip=6.0, x0_hook=None): | |
| """DDIM sampler with classifier-free guidance. | |
| - plain generation : x_init=None | |
| - SDEdit repair : x_init=<clean-ish observed pyramid>, t_start=<noise level> | |
| - inpainting : known=<values>, known_mask=<1 where observed> (RePaint-style) | |
| """ | |
| model.eval() | |
| device = next(model.parameters()).device | |
| diff.to(device) | |
| t_start = diff.T - 1 if t_start is None else int(t_start) | |
| if x_init is None: | |
| B = n if cond is None else cond.shape[0] | |
| if B is None: | |
| B = known.shape[0] | |
| x = torch.randn(B, 2, N_AGES, device=device) | |
| else: | |
| B = x_init.shape[0] | |
| x = diff.q_sample(x_init, torch.full((B,), t_start, device=device, dtype=torch.long))[0] | |
| def predict(xt, t): | |
| tt = torch.full((B,), int(t), device=device, dtype=torch.long) | |
| if cond is None: | |
| return model(xt, tt, None) | |
| ec = model(xt, tt, cond) | |
| if guidance == 1.0: | |
| return ec | |
| eu = model(xt, tt, None) | |
| return eu + guidance * (ec - eu) | |
| ts = [] | |
| for t in np.round(np.linspace(t_start, 0, n_steps)).astype(int): | |
| if not ts or t != ts[-1]: | |
| ts.append(int(t)) | |
| for i, t in enumerate(ts): | |
| t_prev = ts[i + 1] if i + 1 < len(ts) else -1 | |
| eps = predict(x, t) | |
| ab = diff.abar[t] | |
| ab_prev = diff.abar[t_prev] if t_prev >= 0 else torch.tensor(1.0, device=device) | |
| x0 = ((x - (1 - ab).sqrt() * eps) / ab.sqrt()).clamp(-clip, clip) | |
| if x0_hook is not None: # e.g. enforce known window totals on the clean estimate | |
| x0 = x0_hook(x0) | |
| eps = (x - ab.sqrt() * x0) / (1 - ab).sqrt() | |
| sigma = eta * ((1 - ab_prev) / (1 - ab)).sqrt() * (1 - ab / ab_prev).clamp(min=0).sqrt() | |
| direction = (1 - ab_prev - sigma ** 2).clamp(min=0).sqrt() * eps | |
| x = ab_prev.sqrt() * x0 + direction + (sigma * torch.randn_like(x) if eta > 0 else 0.0) | |
| if known is not None: | |
| if t_prev >= 0: | |
| kn = diff.q_sample(known, torch.full((B,), t_prev, device=device, dtype=torch.long))[0] | |
| else: | |
| kn = known | |
| x = known_mask * kn + (1 - known_mask) * x | |
| return x | |
| # ---------------------------------------------------------------------------- | |
| # Training | |
| # ---------------------------------------------------------------------------- | |
| def val_loss(model, diff, X, C, seed=123): | |
| g = torch.Generator(device=X.device).manual_seed(seed) | |
| losses = [] | |
| for t_val in (25, 75, 150, 250, 400): | |
| t = torch.full((X.shape[0],), t_val, device=X.device, dtype=torch.long) | |
| noise = torch.randn(X.shape, generator=g, device=X.device) | |
| xt, _ = diff.q_sample(X, t, noise) | |
| losses.append(F.mse_loss(model(xt, t, C), noise).item()) | |
| return float(np.mean(losses)) | |
| def train(model, diff, x, c, x_val=None, c_val=None, steps=12000, bs=256, lr=3e-4, | |
| p_uncond=0.1, ema_decay=0.999, log_every=500, device=None, seed=0): | |
| device = device or ("cuda" if torch.cuda.is_available() else "cpu") | |
| torch.manual_seed(seed) | |
| model.to(device).train() | |
| diff.to(device) | |
| ema = EMA(model, ema_decay) | |
| ema.shadow.to(device) | |
| X, C = torch.as_tensor(x).to(device), torch.as_tensor(c).to(device) | |
| Xv = torch.as_tensor(x_val).to(device) if x_val is not None else None | |
| Cv = torch.as_tensor(c_val).to(device) if c_val is not None else None | |
| opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4) | |
| history, run = [], [] | |
| for step in range(steps): | |
| lr_now = lr * min(1.0, (step + 1) / 500) * 0.5 * (1 + math.cos(math.pi * step / steps)) | |
| for g in opt.param_groups: | |
| g["lr"] = lr_now | |
| idx = torch.randint(0, X.shape[0], (bs,), device=device) | |
| t = torch.randint(0, diff.T, (bs,), device=device) | |
| xt, noise = diff.q_sample(X[idx], t) | |
| drop = torch.rand(bs, device=device) < p_uncond | |
| loss = F.mse_loss(model(xt, t, C[idx], drop), noise) | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| opt.step() | |
| ema.update(model, step) | |
| run.append(loss.item()) | |
| if (step + 1) % log_every == 0 or step + 1 == steps: | |
| rec = {"step": step + 1, "train_loss": float(np.mean(run))} | |
| run = [] | |
| if Xv is not None: | |
| rec["val_loss_ema"] = val_loss(ema.shadow, diff, Xv, Cv) | |
| history.append(rec) | |
| print(rec) | |
| model.eval() | |
| return ema, history | |
| # ---------------------------------------------------------------------------- | |
| # Age heaping: simulator, index, repair | |
| # ---------------------------------------------------------------------------- | |
| def simulate_heaping(shares, lam, rng=None, digit0_boost=1.6, noise_sd=0.03): | |
| """Mimic digit preference. A fraction `lam` (per pyramid) of people aged within +-2 of a | |
| multiple of 5 report that multiple instead (stronger for multiples of 10), plus mild | |
| multiplicative roster noise. shares: (n, 2, 101); lam: scalar or (n,).""" | |
| rng = np.random.default_rng() if rng is None else rng | |
| out = np.array(shares, dtype=np.float64).reshape(-1, 2, N_AGES) | |
| lam = np.broadcast_to(np.asarray(lam, dtype=np.float64), (len(out),)) | |
| for h in range(5, 100, 5): | |
| boost = digit0_boost if h % 10 == 0 else 1.0 | |
| for d, w in ((-2, 0.6), (-1, 1.0), (1, 1.0), (2, 0.6)): | |
| frac = np.clip(lam * boost * w, 0, 0.9)[:, None] | |
| moved = out[:, :, h + d] * frac | |
| out[:, :, h + d] -= moved | |
| out[:, :, h] += moved | |
| out *= np.exp(rng.normal(0.0, noise_sd, out.shape)) | |
| return out / out.sum(axis=(1, 2), keepdims=True) | |
| def whipple(shares): | |
| """Whipple index (both sexes), ages 23-62. ~100 = no preference, 500 = only 0/5.""" | |
| tot = np.asarray(shares, dtype=np.float64).reshape(-1, 2, N_AGES).sum(axis=1) | |
| ages = np.arange(23, 63) | |
| num = tot[:, ages[ages % 5 == 0]].sum(-1) | |
| den = tot[:, 23:63].sum(-1) | |
| return 500.0 * num / den | |
| def project_to_windows(rep, obs): | |
| """Rescale `rep` so that totals of windows centred on heaping digits match `obs` | |
| (heaping moves people within +-2 years, so these totals are approximately conserved).""" | |
| rep = np.asarray(rep, dtype=np.float64) | |
| obs = np.asarray(obs, dtype=np.float64) | |
| out = rep.copy() | |
| for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]): | |
| r = rep[..., a:b].sum(-1, keepdims=True) | |
| o = obs[..., a:b].sum(-1, keepdims=True) | |
| out[..., a:b] = rep[..., a:b] * (o / np.maximum(r, 1e-12)) | |
| return out | |
| def _norm_shares(a): | |
| a = np.asarray(a, dtype=np.float64).reshape(-1, 2, N_AGES) | |
| return a / a.sum(axis=(1, 2), keepdims=True) | |
| def _decode_t(x, norm): | |
| s = (torch.exp(x * norm.sigma + norm.mu) - norm.eps).clamp(min=0.0) / (2 * N_AGES) | |
| return s / s.sum(dim=(-2, -1), keepdim=True) | |
| def _encode_t(s, norm): | |
| return (torch.log(s * 2 * N_AGES + norm.eps) - norm.mu) / norm.sigma | |
| def _project_t(rep, obs): | |
| out = rep.clone() | |
| for a, b in zip(WINDOW_EDGES[:-1], WINDOW_EDGES[1:]): | |
| r = rep[..., a:b].sum(-1, keepdim=True) | |
| o = obs[..., a:b].sum(-1, keepdim=True) | |
| out[..., a:b] = rep[..., a:b] * (o / r.clamp(min=1e-12)) | |
| return out | |
| def pchip_baseline(obs): | |
| """Classical window-total-preserving smoother: monotone (PCHIP) interpolation of the cumulative | |
| population at the window edges, differenced back to single ages. Heaping cannot move the | |
| window totals much, so this removes digit preference without a learned model.""" | |
| from scipy.interpolate import PchipInterpolator | |
| obs = _norm_shares(obs) | |
| edges = np.array(WINDOW_EDGES) | |
| tot = np.add.reduceat(obs, edges[:-1], axis=-1) # (n, 2, n_windows) | |
| cum = np.concatenate([np.zeros(tot.shape[:-1] + (1,)), np.cumsum(tot, axis=-1)], axis=-1) | |
| f = PchipInterpolator(edges, cum, axis=-1) | |
| out = np.clip(np.diff(f(np.arange(N_AGES + 1)), axis=-1), 0.0, None) | |
| return out / out.sum(axis=(1, 2), keepdims=True) | |
| def repair(model, diff, norm, obs_shares, years, t_start=200, n_steps=40, n_samples=4, | |
| use_cond=True, project="each_step", init="pchip", seed=0): | |
| """Repair heaped pyramids. Returns shares (n, 2, 101). | |
| init : "pchip" starts from the classical window-total-preserving smoothing (in-distribution | |
| input for the model); "obs" starts from the raw heaped pyramid. | |
| project : "each_step" re-imposes the observed window totals on the clean estimate at every | |
| denoising step (the model only decides the within-window shape); "post" projects once | |
| at the end; "none" does not project. | |
| """ | |
| device = next(model.parameters()).device | |
| obs = _norm_shares(obs_shares) | |
| start = pchip_baseline(obs) if init == "pchip" else obs | |
| x_start = torch.as_tensor(norm.encode(start)).to(device) | |
| cond = None | |
| if use_cond: | |
| cond = torch.as_tensor(build_cond(summary_stats(obs), years)).to(device) | |
| hook = None | |
| if project == "each_step": | |
| obs_t = torch.as_tensor(obs, dtype=torch.float32, device=device) | |
| def hook(x0): | |
| return _encode_t(_project_t(_decode_t(x0, norm), obs_t), norm).clamp(-6.0, 6.0) | |
| torch.manual_seed(seed) | |
| outs = [] | |
| for _ in range(n_samples): | |
| x = sample(model, diff, cond, n_steps=n_steps, guidance=1.0, eta=0.0, | |
| x_init=x_start, t_start=t_start, x0_hook=hook) | |
| outs.append(norm.decode(x.cpu().numpy())) | |
| rep = np.mean(outs, axis=0) | |
| if project in ("each_step", "post"): | |
| rep = project_to_windows(rep, obs) | |
| return rep / rep.sum(axis=(1, 2), keepdims=True) | |
| def inpaint(model, diff, norm, obs_shares, observed_mask, years, n_steps=60, guidance=1.5, seed=0): | |
| """Fill unobserved ages. observed_mask: (2,101) or (n,2,101) bool, True = observed. | |
| Values at unobserved positions in obs_shares are ignored (use any positive number).""" | |
| device = next(model.parameters()).device | |
| obs = np.clip(_norm_shares(obs_shares), 1e-9, None) | |
| mask = np.broadcast_to(np.asarray(observed_mask, dtype=bool), obs.shape) | |
| known = torch.as_tensor(norm.encode(obs)).to(device) | |
| m = torch.as_tensor(mask.astype(np.float32)).to(device) | |
| # conditioning from the observed part only: fill the gaps with the mean observed share | |
| filled = np.where(mask, obs, obs[mask].mean()) | |
| cond = torch.as_tensor(build_cond(summary_stats(filled), years)).to(device) | |
| torch.manual_seed(seed) | |
| x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.3, | |
| known=known, known_mask=m) | |
| return norm.decode(x.cpu().numpy()) | |
| def anomaly_score(model, diff, x, cond, ts=(20, 40, 60, 80, 120), seed=0): | |
| """Mean noise-prediction error at low noise levels; higher = less plausible pyramid.""" | |
| model.eval() | |
| diff.to(x.device) | |
| g = torch.Generator(device=x.device).manual_seed(seed) | |
| errs = [] | |
| for t_val in ts: | |
| t = torch.full((x.shape[0],), t_val, device=x.device, dtype=torch.long) | |
| noise = torch.randn(x.shape, generator=g, device=x.device) | |
| xt, _ = diff.q_sample(x, t, noise) | |
| errs.append(((model(xt, t, cond) - noise) ** 2).mean(dim=(1, 2))) | |
| return torch.stack(errs).mean(0).cpu().numpy() | |
| # ---------------------------------------------------------------------------- | |
| # Convenience | |
| # ---------------------------------------------------------------------------- | |
| def generate(model, diff, norm, median_age, u15, p65, sex_ratio, year, n=16, | |
| guidance=2.0, n_steps=50, seed=0): | |
| device = next(model.parameters()).device | |
| stats = np.tile(np.array([[median_age, u15, p65, sex_ratio]], dtype=np.float64), (n, 1)) | |
| cond = torch.as_tensor(build_cond(stats, year)).to(device) | |
| torch.manual_seed(seed) | |
| x = sample(model, diff, cond, n_steps=n_steps, guidance=guidance, eta=0.0) | |
| return norm.decode(x.cpu().numpy()) | |
| def roster_to_pyramid(ages, is_male, weights=None, max_age=100): | |
| """Household-roster microdata -> (2, 101) shares. Drop missing / 'don't know' ages first.""" | |
| ages = np.asarray(ages, dtype=float) | |
| is_male = np.asarray(is_male, dtype=bool) | |
| w = np.ones_like(ages) if weights is None else np.asarray(weights, dtype=float) | |
| ok = np.isfinite(ages) & np.isfinite(w) & (ages >= 0) | |
| ages = np.clip(ages[ok].astype(int), 0, max_age) | |
| is_male, w = is_male[ok], w[ok] | |
| male = np.bincount(ages[is_male], weights=w[is_male], minlength=N_AGES)[:N_AGES] | |
| female = np.bincount(ages[~is_male], weights=w[~is_male], minlength=N_AGES)[:N_AGES] | |
| p = np.stack([male, female]) | |
| return p / p.sum() | |
| def plot_repair(obs, rep, truth=None, title=""): | |
| import matplotlib.pyplot as plt | |
| fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True) | |
| ages = np.arange(N_AGES) | |
| for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])): | |
| if truth is not None: | |
| ax.plot(ages, truth[k] * 1e3, lw=1.2, color="0.5", label="truth") | |
| ax.plot(ages, obs[k] * 1e3, lw=0.9, color="tab:red", alpha=0.8, label="observed") | |
| ax.plot(ages, rep[k] * 1e3, lw=1.4, color="tab:blue", label="repaired") | |
| ax.set_title(name) | |
| ax.set_xlabel("age") | |
| axes[0].set_ylabel("share of population (per mille)") | |
| axes[0].legend(frameon=False) | |
| fig.suptitle(title) | |
| fig.tight_layout() | |
| return fig | |
| def plot_generated(samples, title=""): | |
| import matplotlib.pyplot as plt | |
| fig, axes = plt.subplots(1, 2, figsize=(11, 3.6), sharey=True) | |
| ages = np.arange(N_AGES) | |
| lo, med, hi = np.percentile(samples, [5, 50, 95], axis=0) | |
| for k, (ax, name) in enumerate(zip(axes, ["Male", "Female"])): | |
| ax.fill_between(ages, lo[k] * 1e3, hi[k] * 1e3, alpha=0.25, color="tab:blue", label="5-95% of samples") | |
| ax.plot(ages, med[k] * 1e3, color="tab:blue", label="median sample") | |
| ax.set_title(name) | |
| ax.set_xlabel("age") | |
| axes[0].set_ylabel("share of population (per mille)") | |
| axes[0].legend(frameon=False) | |
| fig.suptitle(title) | |
| fig.tight_layout() | |
| return fig | |
| # ---------------------------------------------------------------------------- | |
| # Evaluation on held-out countries | |
| # ---------------------------------------------------------------------------- | |
| def evaluate_repair(model, diff, norm, shares, years, t_start=200, lam_range=(0.10, 0.35), | |
| seed=0, n_steps=40, n_samples=4, | |
| variants=("obs_post", "obs_each", "pchip_each")): | |
| """Heap held-out pyramids synthetically, repair, and compare with truth and two classical | |
| baselines (5-point moving average and PCHIP window-total interpolation). MAE is in per-mille of | |
| total population (lower = better). Primary PyramidDiff variant: pchip_each.""" | |
| from scipy.ndimage import uniform_filter1d | |
| rng = np.random.default_rng(seed) | |
| truth = _norm_shares(shares) | |
| lam = rng.uniform(*lam_range, size=len(truth)) | |
| obs = simulate_heaping(truth, lam, rng) | |
| ma = uniform_filter1d(obs, size=5, axis=-1, mode="nearest") | |
| ma = project_to_windows(ma / ma.sum(axis=(1, 2), keepdims=True), obs) | |
| ma = ma / ma.sum(axis=(1, 2), keepdims=True) | |
| pc = pchip_baseline(obs) | |
| cfg = {"obs_post": ("pyramiddiff_obs_init_posthoc_projection", dict(init="obs", project="post")), | |
| "obs_each": ("pyramiddiff_obs_init_each_step_projection", dict(init="obs", project="each_step")), | |
| "pchip_each": ("pyramiddiff_pchip_init_each_step_projection", dict(init="pchip", project="each_step"))} | |
| preds = {"observed_heaped": obs, "moving_average_5_projected": ma, "pchip_window_totals": pc} | |
| for v in variants: | |
| name, kw = cfg[v] | |
| preds[name] = repair(model, diff, norm, obs, years, t_start=t_start, n_steps=n_steps, | |
| n_samples=n_samples, seed=seed, **kw) | |
| def mae(a): | |
| return float(np.abs(a - truth).mean() * 1e3) | |
| primary = cfg["pchip_each"][0] if "pchip_each" in variants else cfg[variants[0]][0] | |
| return { | |
| "n_pyramids": int(len(truth)), | |
| "t_start": int(t_start), | |
| "primary": primary, | |
| "mae_permille": {k: mae(v) for k, v in preds.items()}, | |
| "whipple_mean": {"truth": float(whipple(truth).mean()), | |
| **{k: float(whipple(v).mean()) for k, v in preds.items()}}, | |
| "_arrays": {"truth": truth, "obs": obs, "rep": preds[primary], "ma": ma, "pchip": pc}, | |
| } | |
| def evaluate_anomaly(model, diff, norm, shares, years, lam_range=(0.10, 0.35), seed=0): | |
| """AUROC of the denoising-error score for separating clean vs synthetically heaped pyramids.""" | |
| from sklearn.metrics import roc_auc_score | |
| device = next(model.parameters()).device | |
| rng = np.random.default_rng(seed) | |
| truth = _norm_shares(shares) | |
| obs = simulate_heaping(truth, rng.uniform(*lam_range, size=len(truth)), rng) | |
| def score(a): | |
| x = torch.as_tensor(norm.encode(a)).to(device) | |
| c = torch.as_tensor(build_cond(summary_stats(a), years)).to(device) | |
| return anomaly_score(model, diff, x, c) | |
| s_clean, s_heap = score(truth), score(obs) | |
| y = np.r_[np.zeros(len(s_clean)), np.ones(len(s_heap))] | |
| return {"auroc_clean_vs_heaped": float(roc_auc_score(y, np.r_[s_clean, s_heap])), | |
| "score_mean_clean": float(s_clean.mean()), "score_mean_heaped": float(s_heap.mean())} | |
| def evaluate_inpaint(model, diff, norm, shares, years, span=(30, 50), seed=0, n_steps=60, guidance=1.5, | |
| batch=512, max_n=None): | |
| """Hide a contiguous age span (both sexes) of held-out pyramids, fill it with the model (RePaint-style), | |
| and score the hidden ages only (MAE in per mille of total population; lower = better). | |
| Baselines: linear interpolation of the standardised log-share across the gap, and filling with the mean share.""" | |
| rng = np.random.default_rng(seed) | |
| truth = _norm_shares(shares) | |
| years = np.broadcast_to(np.asarray(years), (len(truth),)) | |
| if max_n is not None and len(truth) > max_n: | |
| keep = np.sort(rng.choice(len(truth), max_n, replace=False)) | |
| truth, years = truth[keep], years[keep] | |
| a, b = span | |
| mask = np.ones(truth.shape, dtype=bool) | |
| mask[:, :, a:b] = False | |
| obs = truth.copy() | |
| obs[~mask] = truth[mask].mean() | |
| outs = [] | |
| for i in range(0, len(truth), batch): | |
| outs.append(inpaint(model, diff, norm, obs[i:i + batch], mask[i:i + batch], years[i:i + batch], | |
| n_steps=n_steps, guidance=guidance, seed=seed + i)) | |
| rep = np.concatenate(outs) | |
| z = norm.encode(truth) | |
| zi = z.copy() | |
| w = (np.arange(a, b) - (a - 1)) / (b - (a - 1)) | |
| zi[:, :, a:b] = z[:, :, a - 1:a] * (1 - w) + z[:, :, b:b + 1] * w | |
| lin = norm.decode(zi) | |
| def mae_gap(x): | |
| return float(np.abs(x[:, :, a:b] - truth[:, :, a:b]).mean() * 1e3) | |
| return {"n_pyramids": int(len(truth)), "hidden_ages": [int(a), int(b)], | |
| "mae_permille_hidden_ages": {"pyramiddiff_inpaint": mae_gap(rep), | |
| "linear_interpolation_log_share": mae_gap(lin), | |
| "mean_share_fill": mae_gap(obs)}, | |
| "_arrays": {"truth": truth, "rep": rep, "lin": lin}} | |
| # ---------------------------------------------------------------------------- | |
| # Save / load | |
| # ---------------------------------------------------------------------------- | |
| def save_pretrained(path, model, norm, T, extra=None): | |
| from safetensors.torch import save_file | |
| os.makedirs(path, exist_ok=True) | |
| sd = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()} | |
| save_file(sd, os.path.join(path, "model.safetensors")) | |
| cfg = {"arch": model.cfg, "T": int(T), "n_ages": N_AGES, | |
| "norm": {"mu": norm.mu, "sigma": norm.sigma, "eps": norm.eps}, "extra": extra or {}} | |
| with open(os.path.join(path, "config.json"), "w") as f: | |
| json.dump(cfg, f, indent=2) | |
| def load_pretrained(path, device="cpu"): | |
| from safetensors.torch import load_file | |
| with open(os.path.join(path, "config.json")) as f: | |
| cfg = json.load(f) | |
| model = PyramidUNet(**cfg["arch"]) | |
| model.load_state_dict(load_file(os.path.join(path, "model.safetensors"))) | |
| model.to(device).eval() | |
| diff = Diffusion(cfg["T"], device) | |
| norm = Normaliser(**cfg["norm"]) | |
| return model, diff, norm | |