"""Masked diffusion: training objectives and decoding strategies. The central problem this file exists to study: MDLM training teaches each masked position its MARGINAL p(x_i | observed). Parallel decoding reveals k positions at once by sampling the PRODUCT of those marginals. The truth is the joint, and the gap is exactly the mutual information among the revealed positions. Quality therefore falls as the parallelism rises - which is the only reason to use diffusion in the first place. Every published fix reorders or gates commits at INFERENCE. The model has still never seen a partially-decoded state containing its own correlated mistakes. `self_corrupted_loss` changes that at training time. """ import torch import torch.nn.functional as F # --------------------------------------------------------------- training --- def mdlm_loss(model, x0, tok, t_min=None, fixed_t=None): """Standard MDLM objective: mask each token independently with prob t, predict the originals, weight by 1/t.""" B, L = x0.shape dev = x0.device t_min = t_min if t_min is not None else 1.0 / L t = (torch.full((B,), fixed_t, device=dev) if fixed_t else t_min + (1.0 - t_min) * torch.rand(B, device=dev)) m = torch.rand(B, L, device=dev) < t[:, None] m[torch.arange(B, device=dev), torch.randint(0, L, (B,), device=dev)] = True xt = torch.where(m, torch.full_like(x0, tok.mask), x0) logits = model(xt) ce = F.cross_entropy(logits.reshape(-1, logits.size(-1)), x0.reshape(-1), reduction="none").view(B, L) return ((1.0 / t) * (ce * m).sum(1) / L).mean() def self_corrupted_loss(model, x0, tok, k_frac=0.25, t_min=None, correct_weight=1.0): """OUR METHOD. Train on states that parallel decoding actually produces. 1. mask as usual and take a NO-GRAD pass to get the current marginals 2. reveal a random subset of the masked positions *in parallel*, sampling each independently - i.e. commit exactly the kind of jointly-inconsistent tokens that parallel decoding commits 3. train on the remaining masked positions given that self-generated context, so the model learns p(x_i | observed, its own parallel samples) 4. additionally train it to REPAIR the committed tokens, which turns a wrong commit from an absorbing error into a recoverable one The model is therefore optimised for the distribution it meets at inference rather than for ground-truth context it will never see. """ B, L = x0.shape dev = x0.device t_min = t_min if t_min is not None else 1.0 / L t = t_min + (1.0 - t_min) * torch.rand(B, device=dev) m = torch.rand(B, L, device=dev) < t[:, None] m[torch.arange(B, device=dev), torch.randint(0, L, (B,), device=dev)] = True xt = torch.where(m, torch.full_like(x0, tok.mask), x0) with torch.no_grad(): probs = model(xt).softmax(-1) samp = torch.distributions.Categorical(probs=probs).sample() # commit a random subset of masked positions from the model's own samples commit = m & (torch.rand(B, L, device=dev) < k_frac) x_self = torch.where(commit, samp, xt) still = m & ~commit logits = model(x_self) ce = F.cross_entropy(logits.reshape(-1, logits.size(-1)), x0.reshape(-1), reduction="none").view(B, L) denom = still.sum(1).clamp_min(1) main = ((1.0 / t) * (ce * still).sum(1) / denom).mean() # repair term: committed positions are supervised against ground truth too rep = (ce * commit).sum(1) / commit.sum(1).clamp_min(1) return main + correct_weight * rep.mean() # --------------------------------------------------------------- decoding --- @torch.no_grad() def decode(model, shape, tok, steps, device, strategy="confidence", temperature=0.0, entropy_tau=None, x_init=None, keep_trace=False): """Reveal the canvas over `steps` passes. Fewer passes = more tokens committed per pass = more parallelism = more conditional-independence error. strategies confidence : reveal the most confident masked positions (the standard) random : reveal a random subset (ablation: is confidence doing work?) entropy : reveal only positions whose entropy is below `entropy_tau`, an inference-time baseline for dependency-aware revealing """ B, L = shape x = torch.full((B, L), tok.mask, device=device, dtype=torch.long) if x_init is not None: x = x_init.clone() trace = [] for s in range(steps, 0, -1): masked = x == tok.mask if not masked.any(): break logits = model(x) probs = logits.softmax(-1) if temperature > 0: pred = torch.distributions.Categorical( logits=logits / temperature).sample() conf = probs.gather(-1, pred[..., None]).squeeze(-1) else: conf, pred = probs.max(-1) if strategy == "random": score = torch.rand_like(conf) elif strategy == "entropy": ent = -(probs * probs.clamp_min(1e-9).log()).sum(-1) score = -ent else: score = conf score = score.masked_fill(~masked, -1e9) target_left = int(L * (s - 1) / steps) for b in range(B): nm = int(masked[b].sum()) k = max(0, nm - target_left) if strategy == "entropy" and entropy_tau is not None and k > 0: ent_b = -(probs[b] * probs[b].clamp_min(1e-9).log()).sum(-1) allowed = int(((ent_b < entropy_tau) & masked[b]).sum()) k = max(1, min(k, allowed)) if allowed else 1 if k: idx = score[b].topk(k).indices x[b, idx] = pred[b, idx] if keep_trace: trace.append(x.clone()) still = x == tok.mask if still.any(): x[still] = pred[still] return (x, trace) if keep_trace else x @torch.no_grad() def nll_bpc(model, batches, tok, device, mc=8): """Monte-Carlo estimate of the MDLM bound, reported as bits per character - the standard text8 metric.""" tot, n = 0.0, 0 for x0 in batches: x0 = x0.to(device) B, L = x0.shape for _ in range(mc): t = (1.0 / L) + (1 - 1.0 / L) * torch.rand(B, device=device) m = torch.rand(B, L, device=device) < t[:, None] xt = torch.where(m, torch.full_like(x0, tok.mask), x0) ce = F.cross_entropy(model(xt).reshape(-1, len(tok)), x0.reshape(-1), reduction="none").view(B, L) tot += float(((1.0 / t) * (ce * m).sum(1) / L).sum()) n += B return tot / max(n, 1) / torch.log(torch.tensor(2.0)).item()