texttoimage / t2i_text.py
Murali Chandran
Add CIFAR-10 T2I + SR pipeline
97f6b95
Raw History Blame Contribute Delete
15.6 kB
# label2image_cifar32.py (prompt-based CIFAR-10 diffusion, fixed timestep handling)
# -----------------------------------------------------------
# CIFAR-10 conditional diffusion with free-text prompts (32×32)
# -----------------------------------------------------------
import os, math, argparse, random
from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets, transforms, utils as vutils
from tqdm import tqdm
# ------------------ utils ------------------
def set_seed(seed: int = 0):
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
def _raw_module(m):
return getattr(m, "_orig_mod", m)
CIFAR10_CLASSES = [
"airplane", "automobile", "bird", "cat", "deer",
"dog", "frog", "horse", "ship", "truck"
]
# ------------------ tokenizer ------------------
class TinyTokenizer:
def __init__(self, chars=None, max_len=16):
if chars is None:
chars = list("abcdefghijklmnopqrstuvwxyz0123456789 ,.")
self.stoi = {ch: i+1 for i, ch in enumerate(chars)} # 0 = pad
self.itos = {i+1: ch for i, ch in enumerate(chars)}
self.max_len = max_len
def encode(self, text):
text = text.lower()[:self.max_len]
ids = [self.stoi.get(ch, 0) for ch in text]
return ids + [0]*(self.max_len - len(ids))
def decode(self, ids):
return "".join(self.itos.get(i, "?") for i in ids if i > 0)
# ------------------ text encoder ------------------
class TextEncoder(nn.Module):
def __init__(self, vocab_size, dim, max_len=16, nhead=4, num_layers=2):
super().__init__()
self.emb = nn.Embedding(vocab_size+1, dim)
self.pos = nn.Embedding(max_len, dim)
enc_layer = nn.TransformerEncoderLayer(d_model=dim, nhead=nhead, batch_first=True)
self.enc = nn.TransformerEncoder(enc_layer, num_layers=num_layers)
self.max_len = max_len
def forward(self, tokens):
B, L = tokens.shape
pos = torch.arange(L, device=tokens.device).unsqueeze(0).expand(B, L)
x = self.emb(tokens) + self.pos(pos)
x = self.enc(x)
return x.mean(dim=1)
# ------------------ time embedding ------------------
class SinusoidalTimeEmbedding(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
def forward(self, t):
half = self.dim // 2
emb = torch.log(torch.tensor(10000.0)) / (half - 1)
emb = torch.exp(torch.arange(half, device=t.device) * -emb)
emb = t[:, None].float() * emb[None, :]
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
if self.dim % 2 == 1:
emb = F.pad(emb, (0,1))
return emb
# ------------------ building blocks ------------------
class ResBlock(nn.Module):
def __init__(self, dim_in, dim_out, cond_dim):
super().__init__()
self.norm1 = nn.GroupNorm(32, dim_in)
self.conv1 = nn.Conv2d(dim_in, dim_out, 3, padding=1)
self.norm2 = nn.GroupNorm(32, dim_out)
self.conv2 = nn.Conv2d(dim_out, dim_out, 3, padding=1)
self.emb = nn.Sequential(
nn.SiLU(),
nn.Linear(cond_dim, dim_out * 2)
)
self.act = nn.SiLU()
self.skip = nn.Conv2d(dim_in, dim_out, 1) if dim_in != dim_out else nn.Identity()
def forward(self, x, cond):
h = self.conv1(self.act(self.norm1(x)))
scale, shift = self.emb(cond).chunk(2, dim=1)
scale = scale[:, :, None, None]
shift = shift[:, :, None, None]
h = h * (1 + scale) + shift
h = self.conv2(self.act(self.norm2(h)))
return h + self.skip(x)
class Down(nn.Module):
def __init__(self, dim_in, dim_out, cond_dim):
super().__init__()
self.block1 = ResBlock(dim_in, dim_out, cond_dim)
self.block2 = ResBlock(dim_out, dim_out, cond_dim)
self.down = nn.Conv2d(dim_out, dim_out, 4, stride=2, padding=1)
def forward(self, x, cond):
x = self.block1(x, cond)
x = self.block2(x, cond)
x = self.down(x)
return x
class Up(nn.Module):
def __init__(self, dim_in, dim_out, cond_dim):
super().__init__()
self.block1 = ResBlock(dim_in, dim_out, cond_dim)
self.block2 = ResBlock(dim_out, dim_out, cond_dim)
self.up = nn.ConvTranspose2d(dim_out, dim_out, 4, stride=2, padding=1)
def forward(self, x, cond):
x = self.block1(x, cond)
x = self.block2(x, cond)
x = self.up(x)
return x
class UNetSmall(nn.Module):
def __init__(self, in_ch=3, base=64, cond_dim=256):
super().__init__()
self.in_conv = nn.Conv2d(in_ch, base, 3, padding=1)
self.down1 = Down(base, base*2, cond_dim)
self.down2 = Down(base*2, base*4, cond_dim)
self.mid1 = ResBlock(base*4, base*4, cond_dim)
self.mid2 = ResBlock(base*4, base*4, cond_dim)
self.up2 = Up(base*4, base*2, cond_dim)
self.up1 = Up(base*2, base, cond_dim)
self.out_norm = nn.GroupNorm(8, base)
self.out_conv = nn.Conv2d(base, in_ch, 3, padding=1)
self.act = nn.SiLU()
def forward(self, x, cond):
x0 = self.in_conv(x)
d1 = self.down1(x0, cond)
d2 = self.down2(d1, cond)
m = self.mid1(d2, cond)
m = self.mid2(m, cond)
u2 = self.up2(m + d2, cond)
u1 = self.up1(u2 + d1, cond)
h = self.act(self.out_norm(u1))
return self.out_conv(h)
# ------------------ diffusion ------------------
class Diffusion(nn.Module):
def __init__(self, model: nn.Module, T=1000, schedule="cosine"):
super().__init__()
self.model = model
self.T = T
self.register_buffer("betas", self.make_betas(T, schedule))
alphas = 1.0 - self.betas
alphas_cumprod = torch.cumprod(alphas, dim=0)
self.register_buffer("alphas_cumprod", alphas_cumprod)
self.register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod))
self.register_buffer("sqrt_one_minus_alphas_cumprod", torch.sqrt(1.0 - alphas_cumprod))
from diffusers.schedulers.scheduling_ddpm import DDPMScheduler
self.scheduler = DDPMScheduler(num_train_timesteps=self.T, beta_schedule="squaredcos_cap_v2")
@staticmethod
def make_betas(T, schedule):
if schedule == "linear":
return torch.linspace(1e-4, 0.02, T)
s = 0.008
steps = T + 1
x = torch.linspace(0, T, steps)
alphas_cum = torch.cos(((x / T) + s) / (1 + s) * math.pi * 0.5) ** 2
alphas_cum = alphas_cum / alphas_cum[0]
betas = 1 - (alphas_cum[1:] / alphas_cum[:-1])
return betas.clamp(max=0.999)
@torch.no_grad()
def sample(self, shape, steps=1000, tokens=None):
device = next(self.parameters()).device
x = torch.randn(shape, device=device)
self.scheduler.set_timesteps(steps)
for t in self.scheduler.timesteps:
eps_pred = self.model(x, torch.full((shape[0],), t, device=device, dtype=torch.long), tokens)
x = self.scheduler.step(eps_pred, t, x).prev_sample
return x
# ------------------ EMA ------------------
class EMA:
def __init__(self, model: nn.Module, decay=0.999):
self.decay = decay
self.shadow = {k: v.detach().clone() for k, v in _raw_module(model).state_dict().items()}
def update(self, model: nn.Module):
with torch.no_grad():
for k, v in _raw_module(model).state_dict().items():
if v.dtype.is_floating_point:
self.shadow[k].mul_(self.decay).add_(v, alpha=1 - self.decay)
def copy_to(self, model: nn.Module):
with torch.no_grad():
for k, v in _raw_module(model).state_dict().items():
if k in self.shadow:
v.copy_(self.shadow[k])
# ------------------ Wrapper model ------------------
class CondUNet(nn.Module):
def __init__(self, base=64, time_dim=128, cond_dim=256, vocab_size=40, max_len=16):
super().__init__()
self.time_emb = SinusoidalTimeEmbedding(time_dim)
self.time_mlp = nn.Sequential(
nn.Linear(time_dim, cond_dim), nn.SiLU(), nn.Linear(cond_dim, cond_dim)
)
self.text_enc = TextEncoder(vocab_size, cond_dim, max_len=max_len)
self.net = UNetSmall(in_ch=3, base=base, cond_dim=cond_dim)
def forward(self, x, t, tokens=None):
t_emb = self.time_mlp(self.time_emb(t))
if tokens is not None:
txt_emb = self.text_enc(tokens)
cond = t_emb + txt_emb
else:
cond = t_emb
return self.net(x, cond)
# ------------------ Trainer ------------------
class Trainer:
def __init__(self, args):
self.args = args
set_seed(args.seed)
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tfm = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x * 2 - 1),
])
self.ds = datasets.CIFAR10(args.data_root, train=True, transform=tfm, download=True)
self.loader = DataLoader(self.ds, batch_size=args.bs, shuffle=True, num_workers=args.workers, pin_memory=True, drop_last=True)
self.tokenizer = TinyTokenizer()
self.model = CondUNet(base=args.base, time_dim=128, cond_dim=args.cond_dim, vocab_size=len(self.tokenizer.stoi), max_len=self.tokenizer.max_len).to(self.device)
self.diff = Diffusion(self.model, T=args.T, schedule=args.schedule).to(self.device)
self.opt = torch.optim.AdamW(self.model.parameters(), lr=args.lr)
self.ema = EMA(self.model, decay=0.999) if args.use_ema else None
os.makedirs(args.outdir, exist_ok=True)
self.ckpt_path = os.path.join(args.outdir, "ckpt_latest.pt")
if os.path.exists(self.ckpt_path):
print(f"[Trainer] Resuming from {self.ckpt_path}")
self.load_ckpt(self.ckpt_path)
def save_ckpt(self):
obj = {
"model": _raw_module(self.model).state_dict(),
"ema": self.ema.shadow if self.ema else None,
"args": vars(self.args),
}
torch.save(obj, self.ckpt_path)
def load_ckpt(self, path: str):
obj = torch.load(path, map_location=self.device)
_raw_module(self.model).load_state_dict(obj["model"])
if self.ema and obj.get("ema"):
self.ema.shadow = obj["ema"]
def train(self):
self.model.train()
for epoch in range(self.args.epochs):
losses = []
pbar = tqdm(self.loader, desc=f"Epoch {epoch+1}/{self.args.epochs}")
for x, y in pbar:
x, y = x.to(self.device), y.to(self.device)
t = torch.randint(0, self.args.T, (x.size(0),), device=self.device)
noise = torch.randn_like(x)
a_t = self.diff.sqrt_alphas_cumprod[t][:, None, None, None]
am1_t = self.diff.sqrt_one_minus_alphas_cumprod[t][:, None, None, None]
x_t = a_t * x + am1_t * noise
texts = [CIFAR10_CLASSES[y_i] for y_i in y]
tokens = torch.tensor([self.tokenizer.encode(txt) for txt in texts], device=self.device)
pred = self.model(x_t, t, tokens)
loss = F.mse_loss(pred, noise)
self.opt.zero_grad(set_to_none=True)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
self.opt.step()
if self.ema:
self.ema.update(self.model)
losses.append(loss.item())
pbar.set_postfix(loss=loss.item())
avg_loss = sum(losses) / len(losses)
print(f"Epoch {epoch+1}/{self.args.epochs} | Avg Loss: {avg_loss:.4f}")
if (epoch + 1) % max(1, self.args.sample_every) == 0:
self.sample_and_save_grid(epoch)
self.save_ckpt()
@torch.no_grad()
def sample_and_save_grid(self, epoch):
self.model.eval()
if self.ema:
shadow = {k: v.clone() for k, v in self.ema.shadow.items()}
bak = {k: v.clone() for k, v in _raw_module(self.model).state_dict().items()}
_raw_module(self.model).load_state_dict(shadow)
prompts = ["dog", "ship on water", "cat", "airplane"]
tokens = torch.tensor([self.tokenizer.encode(p) for p in prompts], device=self.device)
imgs = self.diff.sample((len(prompts), 3, 32, 32), steps=self.args.steps, tokens=tokens)
if self.ema:
_raw_module(self.model).load_state_dict(bak)
imgs = (imgs.clamp(-1, 1) + 1) * 0.5
grid = vutils.make_grid(imgs, nrow=4)
out_path = os.path.join(self.args.outdir, f"sample_epoch{epoch+1}.png")
vutils.save_image(grid, out_path)
print(f"Saved {out_path}")
@torch.no_grad()
def sample(self, prompts=None, steps=50, n_per_prompt=1):
"""
Generate CIFAR-10 32×32 images from text prompts.
Returns tensor in [0,1].
"""
self.model.eval()
if prompts is None:
prompts = ["cat", "dog"] # fallback
tokens = torch.tensor(
[self.tokenizer.encode(p) for p in prompts for _ in range(n_per_prompt)],
device=self.device
)
imgs = self.diff.sample(
(len(prompts) * n_per_prompt, 3, 32, 32),
steps=steps,
tokens=tokens
)
imgs = (imgs.clamp(-1, 1) + 1) * 0.5
return imgs
# ------------------ CLI ------------------
def build_parser():
p = argparse.ArgumentParser(description="Prompt-based CIFAR-10 Diffusion (32×32)")
p.add_argument("--train", action="store_true")
p.add_argument("--sample", action="store_true")
p.add_argument("--data_root", type=str, default="./data")
p.add_argument("--epochs", type=int, default=20)
p.add_argument("--bs", type=int, default=128)
p.add_argument("--lr", type=float, default=1e-4)
p.add_argument("--T", type=int, default=1000)
p.add_argument("--schedule", type=str, default="cosine", choices=["cosine", "linear"])
p.add_argument("--base", type=int, default=64)
p.add_argument("--cond_dim", type=int, default=256)
p.add_argument("--workers", type=int, default=4)
p.add_argument("--seed", type=int, default=0)
p.add_argument("--outdir", type=str, default="outputs")
p.add_argument("--use_ema", action="store_true")
p.add_argument("--sample_every", type=int, default=5)
p.add_argument("--steps", type=int, default=50)
p.add_argument("--prompts", nargs="*", default=["dog", "cat", "ship", "airplane"])
return p
Trainer = Trainer
def main():
args = build_parser().parse_args()
if not args.train and not args.sample:
print("Nothing to do: use --train and/or --sample")
return
if args.train:
tr = Trainer(args)
tr.train()
if args.sample:
tr = Trainer(args)
tr.load_ckpt(tr.ckpt_path)
tokens = torch.tensor([tr.tokenizer.encode(p) for p in args.prompts], device=tr.device)
imgs = tr.diff.sample((len(args.prompts), 3, 32, 32), steps=args.steps, tokens=tokens)
imgs = (imgs.clamp(-1, 1) + 1) * 0.5
grid = vutils.make_grid(imgs, nrow=4)
os.makedirs(args.outdir, exist_ok=True)
out_path = os.path.join(args.outdir, "samples.png")
vutils.save_image(grid, out_path)
print(f"Saved {out_path}")
if __name__ == "__main__":
main()