# label2image_cifar32.py (prompt-based CIFAR-10 diffusion, fixed timestep handling) # ----------------------------------------------------------- # CIFAR-10 conditional diffusion with free-text prompts (32×32) # ----------------------------------------------------------- import os, math, argparse, random from typing import Optional import torch import torch.nn as nn import torch.nn.functional as F from torch.utils.data import DataLoader from torchvision import datasets, transforms, utils as vutils from tqdm import tqdm # ------------------ utils ------------------ def set_seed(seed: int = 0): random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) def _raw_module(m): return getattr(m, "_orig_mod", m) CIFAR10_CLASSES = [ "airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck" ] # ------------------ tokenizer ------------------ class TinyTokenizer: def __init__(self, chars=None, max_len=16): if chars is None: chars = list("abcdefghijklmnopqrstuvwxyz0123456789 ,.") self.stoi = {ch: i+1 for i, ch in enumerate(chars)} # 0 = pad self.itos = {i+1: ch for i, ch in enumerate(chars)} self.max_len = max_len def encode(self, text): text = text.lower()[:self.max_len] ids = [self.stoi.get(ch, 0) for ch in text] return ids + [0]*(self.max_len - len(ids)) def decode(self, ids): return "".join(self.itos.get(i, "?") for i in ids if i > 0) # ------------------ text encoder ------------------ class TextEncoder(nn.Module): def __init__(self, vocab_size, dim, max_len=16, nhead=4, num_layers=2): super().__init__() self.emb = nn.Embedding(vocab_size+1, dim) self.pos = nn.Embedding(max_len, dim) enc_layer = nn.TransformerEncoderLayer(d_model=dim, nhead=nhead, batch_first=True) self.enc = nn.TransformerEncoder(enc_layer, num_layers=num_layers) self.max_len = max_len def forward(self, tokens): B, L = tokens.shape pos = torch.arange(L, device=tokens.device).unsqueeze(0).expand(B, L) x = self.emb(tokens) + self.pos(pos) x = self.enc(x) return x.mean(dim=1) # ------------------ time embedding ------------------ class SinusoidalTimeEmbedding(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, t): half = self.dim // 2 emb = torch.log(torch.tensor(10000.0)) / (half - 1) emb = torch.exp(torch.arange(half, device=t.device) * -emb) emb = t[:, None].float() * emb[None, :] emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) if self.dim % 2 == 1: emb = F.pad(emb, (0,1)) return emb # ------------------ building blocks ------------------ class ResBlock(nn.Module): def __init__(self, dim_in, dim_out, cond_dim): super().__init__() self.norm1 = nn.GroupNorm(32, dim_in) self.conv1 = nn.Conv2d(dim_in, dim_out, 3, padding=1) self.norm2 = nn.GroupNorm(32, dim_out) self.conv2 = nn.Conv2d(dim_out, dim_out, 3, padding=1) self.emb = nn.Sequential( nn.SiLU(), nn.Linear(cond_dim, dim_out * 2) ) self.act = nn.SiLU() self.skip = nn.Conv2d(dim_in, dim_out, 1) if dim_in != dim_out else nn.Identity() def forward(self, x, cond): h = self.conv1(self.act(self.norm1(x))) scale, shift = self.emb(cond).chunk(2, dim=1) scale = scale[:, :, None, None] shift = shift[:, :, None, None] h = h * (1 + scale) + shift h = self.conv2(self.act(self.norm2(h))) return h + self.skip(x) class Down(nn.Module): def __init__(self, dim_in, dim_out, cond_dim): super().__init__() self.block1 = ResBlock(dim_in, dim_out, cond_dim) self.block2 = ResBlock(dim_out, dim_out, cond_dim) self.down = nn.Conv2d(dim_out, dim_out, 4, stride=2, padding=1) def forward(self, x, cond): x = self.block1(x, cond) x = self.block2(x, cond) x = self.down(x) return x class Up(nn.Module): def __init__(self, dim_in, dim_out, cond_dim): super().__init__() self.block1 = ResBlock(dim_in, dim_out, cond_dim) self.block2 = ResBlock(dim_out, dim_out, cond_dim) self.up = nn.ConvTranspose2d(dim_out, dim_out, 4, stride=2, padding=1) def forward(self, x, cond): x = self.block1(x, cond) x = self.block2(x, cond) x = self.up(x) return x class UNetSmall(nn.Module): def __init__(self, in_ch=3, base=64, cond_dim=256): super().__init__() self.in_conv = nn.Conv2d(in_ch, base, 3, padding=1) self.down1 = Down(base, base*2, cond_dim) self.down2 = Down(base*2, base*4, cond_dim) self.mid1 = ResBlock(base*4, base*4, cond_dim) self.mid2 = ResBlock(base*4, base*4, cond_dim) self.up2 = Up(base*4, base*2, cond_dim) self.up1 = Up(base*2, base, cond_dim) self.out_norm = nn.GroupNorm(8, base) self.out_conv = nn.Conv2d(base, in_ch, 3, padding=1) self.act = nn.SiLU() def forward(self, x, cond): x0 = self.in_conv(x) d1 = self.down1(x0, cond) d2 = self.down2(d1, cond) m = self.mid1(d2, cond) m = self.mid2(m, cond) u2 = self.up2(m + d2, cond) u1 = self.up1(u2 + d1, cond) h = self.act(self.out_norm(u1)) return self.out_conv(h) # ------------------ diffusion ------------------ class Diffusion(nn.Module): def __init__(self, model: nn.Module, T=1000, schedule="cosine"): super().__init__() self.model = model self.T = T self.register_buffer("betas", self.make_betas(T, schedule)) alphas = 1.0 - self.betas alphas_cumprod = torch.cumprod(alphas, dim=0) self.register_buffer("alphas_cumprod", alphas_cumprod) self.register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod)) self.register_buffer("sqrt_one_minus_alphas_cumprod", torch.sqrt(1.0 - alphas_cumprod)) from diffusers.schedulers.scheduling_ddpm import DDPMScheduler self.scheduler = DDPMScheduler(num_train_timesteps=self.T, beta_schedule="squaredcos_cap_v2") @staticmethod def make_betas(T, schedule): if schedule == "linear": return torch.linspace(1e-4, 0.02, T) s = 0.008 steps = T + 1 x = torch.linspace(0, T, steps) alphas_cum = torch.cos(((x / T) + s) / (1 + s) * math.pi * 0.5) ** 2 alphas_cum = alphas_cum / alphas_cum[0] betas = 1 - (alphas_cum[1:] / alphas_cum[:-1]) return betas.clamp(max=0.999) @torch.no_grad() def sample(self, shape, steps=1000, tokens=None): device = next(self.parameters()).device x = torch.randn(shape, device=device) self.scheduler.set_timesteps(steps) for t in self.scheduler.timesteps: eps_pred = self.model(x, torch.full((shape[0],), t, device=device, dtype=torch.long), tokens) x = self.scheduler.step(eps_pred, t, x).prev_sample return x # ------------------ EMA ------------------ class EMA: def __init__(self, model: nn.Module, decay=0.999): self.decay = decay self.shadow = {k: v.detach().clone() for k, v in _raw_module(model).state_dict().items()} def update(self, model: nn.Module): with torch.no_grad(): for k, v in _raw_module(model).state_dict().items(): if v.dtype.is_floating_point: self.shadow[k].mul_(self.decay).add_(v, alpha=1 - self.decay) def copy_to(self, model: nn.Module): with torch.no_grad(): for k, v in _raw_module(model).state_dict().items(): if k in self.shadow: v.copy_(self.shadow[k]) # ------------------ Wrapper model ------------------ class CondUNet(nn.Module): def __init__(self, base=64, time_dim=128, cond_dim=256, vocab_size=40, max_len=16): super().__init__() self.time_emb = SinusoidalTimeEmbedding(time_dim) self.time_mlp = nn.Sequential( nn.Linear(time_dim, cond_dim), nn.SiLU(), nn.Linear(cond_dim, cond_dim) ) self.text_enc = TextEncoder(vocab_size, cond_dim, max_len=max_len) self.net = UNetSmall(in_ch=3, base=base, cond_dim=cond_dim) def forward(self, x, t, tokens=None): t_emb = self.time_mlp(self.time_emb(t)) if tokens is not None: txt_emb = self.text_enc(tokens) cond = t_emb + txt_emb else: cond = t_emb return self.net(x, cond) # ------------------ Trainer ------------------ class Trainer: def __init__(self, args): self.args = args set_seed(args.seed) self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tfm = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x * 2 - 1), ]) self.ds = datasets.CIFAR10(args.data_root, train=True, transform=tfm, download=True) self.loader = DataLoader(self.ds, batch_size=args.bs, shuffle=True, num_workers=args.workers, pin_memory=True, drop_last=True) self.tokenizer = TinyTokenizer() self.model = CondUNet(base=args.base, time_dim=128, cond_dim=args.cond_dim, vocab_size=len(self.tokenizer.stoi), max_len=self.tokenizer.max_len).to(self.device) self.diff = Diffusion(self.model, T=args.T, schedule=args.schedule).to(self.device) self.opt = torch.optim.AdamW(self.model.parameters(), lr=args.lr) self.ema = EMA(self.model, decay=0.999) if args.use_ema else None os.makedirs(args.outdir, exist_ok=True) self.ckpt_path = os.path.join(args.outdir, "ckpt_latest.pt") if os.path.exists(self.ckpt_path): print(f"[Trainer] Resuming from {self.ckpt_path}") self.load_ckpt(self.ckpt_path) def save_ckpt(self): obj = { "model": _raw_module(self.model).state_dict(), "ema": self.ema.shadow if self.ema else None, "args": vars(self.args), } torch.save(obj, self.ckpt_path) def load_ckpt(self, path: str): obj = torch.load(path, map_location=self.device) _raw_module(self.model).load_state_dict(obj["model"]) if self.ema and obj.get("ema"): self.ema.shadow = obj["ema"] def train(self): self.model.train() for epoch in range(self.args.epochs): losses = [] pbar = tqdm(self.loader, desc=f"Epoch {epoch+1}/{self.args.epochs}") for x, y in pbar: x, y = x.to(self.device), y.to(self.device) t = torch.randint(0, self.args.T, (x.size(0),), device=self.device) noise = torch.randn_like(x) a_t = self.diff.sqrt_alphas_cumprod[t][:, None, None, None] am1_t = self.diff.sqrt_one_minus_alphas_cumprod[t][:, None, None, None] x_t = a_t * x + am1_t * noise texts = [CIFAR10_CLASSES[y_i] for y_i in y] tokens = torch.tensor([self.tokenizer.encode(txt) for txt in texts], device=self.device) pred = self.model(x_t, t, tokens) loss = F.mse_loss(pred, noise) self.opt.zero_grad(set_to_none=True) loss.backward() nn.utils.clip_grad_norm_(self.model.parameters(), 1.0) self.opt.step() if self.ema: self.ema.update(self.model) losses.append(loss.item()) pbar.set_postfix(loss=loss.item()) avg_loss = sum(losses) / len(losses) print(f"Epoch {epoch+1}/{self.args.epochs} | Avg Loss: {avg_loss:.4f}") if (epoch + 1) % max(1, self.args.sample_every) == 0: self.sample_and_save_grid(epoch) self.save_ckpt() @torch.no_grad() def sample_and_save_grid(self, epoch): self.model.eval() if self.ema: shadow = {k: v.clone() for k, v in self.ema.shadow.items()} bak = {k: v.clone() for k, v in _raw_module(self.model).state_dict().items()} _raw_module(self.model).load_state_dict(shadow) prompts = ["dog", "ship on water", "cat", "airplane"] tokens = torch.tensor([self.tokenizer.encode(p) for p in prompts], device=self.device) imgs = self.diff.sample((len(prompts), 3, 32, 32), steps=self.args.steps, tokens=tokens) if self.ema: _raw_module(self.model).load_state_dict(bak) imgs = (imgs.clamp(-1, 1) + 1) * 0.5 grid = vutils.make_grid(imgs, nrow=4) out_path = os.path.join(self.args.outdir, f"sample_epoch{epoch+1}.png") vutils.save_image(grid, out_path) print(f"Saved {out_path}") @torch.no_grad() def sample(self, prompts=None, steps=50, n_per_prompt=1): """ Generate CIFAR-10 32×32 images from text prompts. Returns tensor in [0,1]. """ self.model.eval() if prompts is None: prompts = ["cat", "dog"] # fallback tokens = torch.tensor( [self.tokenizer.encode(p) for p in prompts for _ in range(n_per_prompt)], device=self.device ) imgs = self.diff.sample( (len(prompts) * n_per_prompt, 3, 32, 32), steps=steps, tokens=tokens ) imgs = (imgs.clamp(-1, 1) + 1) * 0.5 return imgs # ------------------ CLI ------------------ def build_parser(): p = argparse.ArgumentParser(description="Prompt-based CIFAR-10 Diffusion (32×32)") p.add_argument("--train", action="store_true") p.add_argument("--sample", action="store_true") p.add_argument("--data_root", type=str, default="./data") p.add_argument("--epochs", type=int, default=20) p.add_argument("--bs", type=int, default=128) p.add_argument("--lr", type=float, default=1e-4) p.add_argument("--T", type=int, default=1000) p.add_argument("--schedule", type=str, default="cosine", choices=["cosine", "linear"]) p.add_argument("--base", type=int, default=64) p.add_argument("--cond_dim", type=int, default=256) p.add_argument("--workers", type=int, default=4) p.add_argument("--seed", type=int, default=0) p.add_argument("--outdir", type=str, default="outputs") p.add_argument("--use_ema", action="store_true") p.add_argument("--sample_every", type=int, default=5) p.add_argument("--steps", type=int, default=50) p.add_argument("--prompts", nargs="*", default=["dog", "cat", "ship", "airplane"]) return p Trainer = Trainer def main(): args = build_parser().parse_args() if not args.train and not args.sample: print("Nothing to do: use --train and/or --sample") return if args.train: tr = Trainer(args) tr.train() if args.sample: tr = Trainer(args) tr.load_ckpt(tr.ckpt_path) tokens = torch.tensor([tr.tokenizer.encode(p) for p in args.prompts], device=tr.device) imgs = tr.diff.sample((len(args.prompts), 3, 32, 32), steps=args.steps, tokens=tokens) imgs = (imgs.clamp(-1, 1) + 1) * 0.5 grid = vutils.make_grid(imgs, nrow=4) os.makedirs(args.outdir, exist_ok=True) out_path = os.path.join(args.outdir, "samples.png") vutils.save_image(grid, out_path) print(f"Saved {out_path}") if __name__ == "__main__": main()