Spaces:
Runtime error
Runtime error
Download t2i_text.py from codeninja3d/texttoimage: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/spaces/codeninja3d/texttoimage/resolve/main/t2i_text.py
- Command line
-
hf download hf://spaces/codeninja3d/texttoimage/t2i_text.py
-
curl -L -o t2i_text.py https://huggingface.co/spaces/codeninja3d/texttoimage/resolve/main/t2i_text.py
15.6 kB
| # label2image_cifar32.py (prompt-based CIFAR-10 diffusion, fixed timestep handling) | |
| # ----------------------------------------------------------- | |
| # CIFAR-10 conditional diffusion with free-text prompts (32×32) | |
| # ----------------------------------------------------------- | |
| import os, math, argparse, random | |
| from typing import Optional | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.utils.data import DataLoader | |
| from torchvision import datasets, transforms, utils as vutils | |
| from tqdm import tqdm | |
| # ------------------ utils ------------------ | |
| def set_seed(seed: int = 0): | |
| random.seed(seed) | |
| torch.manual_seed(seed) | |
| torch.cuda.manual_seed_all(seed) | |
| def _raw_module(m): | |
| return getattr(m, "_orig_mod", m) | |
| CIFAR10_CLASSES = [ | |
| "airplane", "automobile", "bird", "cat", "deer", | |
| "dog", "frog", "horse", "ship", "truck" | |
| ] | |
| # ------------------ tokenizer ------------------ | |
| class TinyTokenizer: | |
| def __init__(self, chars=None, max_len=16): | |
| if chars is None: | |
| chars = list("abcdefghijklmnopqrstuvwxyz0123456789 ,.") | |
| self.stoi = {ch: i+1 for i, ch in enumerate(chars)} # 0 = pad | |
| self.itos = {i+1: ch for i, ch in enumerate(chars)} | |
| self.max_len = max_len | |
| def encode(self, text): | |
| text = text.lower()[:self.max_len] | |
| ids = [self.stoi.get(ch, 0) for ch in text] | |
| return ids + [0]*(self.max_len - len(ids)) | |
| def decode(self, ids): | |
| return "".join(self.itos.get(i, "?") for i in ids if i > 0) | |
| # ------------------ text encoder ------------------ | |
| class TextEncoder(nn.Module): | |
| def __init__(self, vocab_size, dim, max_len=16, nhead=4, num_layers=2): | |
| super().__init__() | |
| self.emb = nn.Embedding(vocab_size+1, dim) | |
| self.pos = nn.Embedding(max_len, dim) | |
| enc_layer = nn.TransformerEncoderLayer(d_model=dim, nhead=nhead, batch_first=True) | |
| self.enc = nn.TransformerEncoder(enc_layer, num_layers=num_layers) | |
| self.max_len = max_len | |
| def forward(self, tokens): | |
| B, L = tokens.shape | |
| pos = torch.arange(L, device=tokens.device).unsqueeze(0).expand(B, L) | |
| x = self.emb(tokens) + self.pos(pos) | |
| x = self.enc(x) | |
| return x.mean(dim=1) | |
| # ------------------ time embedding ------------------ | |
| class SinusoidalTimeEmbedding(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| def forward(self, t): | |
| half = self.dim // 2 | |
| emb = torch.log(torch.tensor(10000.0)) / (half - 1) | |
| emb = torch.exp(torch.arange(half, device=t.device) * -emb) | |
| emb = t[:, None].float() * emb[None, :] | |
| emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) | |
| if self.dim % 2 == 1: | |
| emb = F.pad(emb, (0,1)) | |
| return emb | |
| # ------------------ building blocks ------------------ | |
| class ResBlock(nn.Module): | |
| def __init__(self, dim_in, dim_out, cond_dim): | |
| super().__init__() | |
| self.norm1 = nn.GroupNorm(32, dim_in) | |
| self.conv1 = nn.Conv2d(dim_in, dim_out, 3, padding=1) | |
| self.norm2 = nn.GroupNorm(32, dim_out) | |
| self.conv2 = nn.Conv2d(dim_out, dim_out, 3, padding=1) | |
| self.emb = nn.Sequential( | |
| nn.SiLU(), | |
| nn.Linear(cond_dim, dim_out * 2) | |
| ) | |
| self.act = nn.SiLU() | |
| self.skip = nn.Conv2d(dim_in, dim_out, 1) if dim_in != dim_out else nn.Identity() | |
| def forward(self, x, cond): | |
| h = self.conv1(self.act(self.norm1(x))) | |
| scale, shift = self.emb(cond).chunk(2, dim=1) | |
| scale = scale[:, :, None, None] | |
| shift = shift[:, :, None, None] | |
| h = h * (1 + scale) + shift | |
| h = self.conv2(self.act(self.norm2(h))) | |
| return h + self.skip(x) | |
| class Down(nn.Module): | |
| def __init__(self, dim_in, dim_out, cond_dim): | |
| super().__init__() | |
| self.block1 = ResBlock(dim_in, dim_out, cond_dim) | |
| self.block2 = ResBlock(dim_out, dim_out, cond_dim) | |
| self.down = nn.Conv2d(dim_out, dim_out, 4, stride=2, padding=1) | |
| def forward(self, x, cond): | |
| x = self.block1(x, cond) | |
| x = self.block2(x, cond) | |
| x = self.down(x) | |
| return x | |
| class Up(nn.Module): | |
| def __init__(self, dim_in, dim_out, cond_dim): | |
| super().__init__() | |
| self.block1 = ResBlock(dim_in, dim_out, cond_dim) | |
| self.block2 = ResBlock(dim_out, dim_out, cond_dim) | |
| self.up = nn.ConvTranspose2d(dim_out, dim_out, 4, stride=2, padding=1) | |
| def forward(self, x, cond): | |
| x = self.block1(x, cond) | |
| x = self.block2(x, cond) | |
| x = self.up(x) | |
| return x | |
| class UNetSmall(nn.Module): | |
| def __init__(self, in_ch=3, base=64, cond_dim=256): | |
| super().__init__() | |
| self.in_conv = nn.Conv2d(in_ch, base, 3, padding=1) | |
| self.down1 = Down(base, base*2, cond_dim) | |
| self.down2 = Down(base*2, base*4, cond_dim) | |
| self.mid1 = ResBlock(base*4, base*4, cond_dim) | |
| self.mid2 = ResBlock(base*4, base*4, cond_dim) | |
| self.up2 = Up(base*4, base*2, cond_dim) | |
| self.up1 = Up(base*2, base, cond_dim) | |
| self.out_norm = nn.GroupNorm(8, base) | |
| self.out_conv = nn.Conv2d(base, in_ch, 3, padding=1) | |
| self.act = nn.SiLU() | |
| def forward(self, x, cond): | |
| x0 = self.in_conv(x) | |
| d1 = self.down1(x0, cond) | |
| d2 = self.down2(d1, cond) | |
| m = self.mid1(d2, cond) | |
| m = self.mid2(m, cond) | |
| u2 = self.up2(m + d2, cond) | |
| u1 = self.up1(u2 + d1, cond) | |
| h = self.act(self.out_norm(u1)) | |
| return self.out_conv(h) | |
| # ------------------ diffusion ------------------ | |
| class Diffusion(nn.Module): | |
| def __init__(self, model: nn.Module, T=1000, schedule="cosine"): | |
| super().__init__() | |
| self.model = model | |
| self.T = T | |
| self.register_buffer("betas", self.make_betas(T, schedule)) | |
| alphas = 1.0 - self.betas | |
| alphas_cumprod = torch.cumprod(alphas, dim=0) | |
| self.register_buffer("alphas_cumprod", alphas_cumprod) | |
| self.register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod)) | |
| self.register_buffer("sqrt_one_minus_alphas_cumprod", torch.sqrt(1.0 - alphas_cumprod)) | |
| from diffusers.schedulers.scheduling_ddpm import DDPMScheduler | |
| self.scheduler = DDPMScheduler(num_train_timesteps=self.T, beta_schedule="squaredcos_cap_v2") | |
| def make_betas(T, schedule): | |
| if schedule == "linear": | |
| return torch.linspace(1e-4, 0.02, T) | |
| s = 0.008 | |
| steps = T + 1 | |
| x = torch.linspace(0, T, steps) | |
| alphas_cum = torch.cos(((x / T) + s) / (1 + s) * math.pi * 0.5) ** 2 | |
| alphas_cum = alphas_cum / alphas_cum[0] | |
| betas = 1 - (alphas_cum[1:] / alphas_cum[:-1]) | |
| return betas.clamp(max=0.999) | |
| def sample(self, shape, steps=1000, tokens=None): | |
| device = next(self.parameters()).device | |
| x = torch.randn(shape, device=device) | |
| self.scheduler.set_timesteps(steps) | |
| for t in self.scheduler.timesteps: | |
| eps_pred = self.model(x, torch.full((shape[0],), t, device=device, dtype=torch.long), tokens) | |
| x = self.scheduler.step(eps_pred, t, x).prev_sample | |
| return x | |
| # ------------------ EMA ------------------ | |
| class EMA: | |
| def __init__(self, model: nn.Module, decay=0.999): | |
| self.decay = decay | |
| self.shadow = {k: v.detach().clone() for k, v in _raw_module(model).state_dict().items()} | |
| def update(self, model: nn.Module): | |
| with torch.no_grad(): | |
| for k, v in _raw_module(model).state_dict().items(): | |
| if v.dtype.is_floating_point: | |
| self.shadow[k].mul_(self.decay).add_(v, alpha=1 - self.decay) | |
| def copy_to(self, model: nn.Module): | |
| with torch.no_grad(): | |
| for k, v in _raw_module(model).state_dict().items(): | |
| if k in self.shadow: | |
| v.copy_(self.shadow[k]) | |
| # ------------------ Wrapper model ------------------ | |
| class CondUNet(nn.Module): | |
| def __init__(self, base=64, time_dim=128, cond_dim=256, vocab_size=40, max_len=16): | |
| super().__init__() | |
| self.time_emb = SinusoidalTimeEmbedding(time_dim) | |
| self.time_mlp = nn.Sequential( | |
| nn.Linear(time_dim, cond_dim), nn.SiLU(), nn.Linear(cond_dim, cond_dim) | |
| ) | |
| self.text_enc = TextEncoder(vocab_size, cond_dim, max_len=max_len) | |
| self.net = UNetSmall(in_ch=3, base=base, cond_dim=cond_dim) | |
| def forward(self, x, t, tokens=None): | |
| t_emb = self.time_mlp(self.time_emb(t)) | |
| if tokens is not None: | |
| txt_emb = self.text_enc(tokens) | |
| cond = t_emb + txt_emb | |
| else: | |
| cond = t_emb | |
| return self.net(x, cond) | |
| # ------------------ Trainer ------------------ | |
| class Trainer: | |
| def __init__(self, args): | |
| self.args = args | |
| set_seed(args.seed) | |
| self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| tfm = transforms.Compose([ | |
| transforms.ToTensor(), | |
| transforms.Lambda(lambda x: x * 2 - 1), | |
| ]) | |
| self.ds = datasets.CIFAR10(args.data_root, train=True, transform=tfm, download=True) | |
| self.loader = DataLoader(self.ds, batch_size=args.bs, shuffle=True, num_workers=args.workers, pin_memory=True, drop_last=True) | |
| self.tokenizer = TinyTokenizer() | |
| self.model = CondUNet(base=args.base, time_dim=128, cond_dim=args.cond_dim, vocab_size=len(self.tokenizer.stoi), max_len=self.tokenizer.max_len).to(self.device) | |
| self.diff = Diffusion(self.model, T=args.T, schedule=args.schedule).to(self.device) | |
| self.opt = torch.optim.AdamW(self.model.parameters(), lr=args.lr) | |
| self.ema = EMA(self.model, decay=0.999) if args.use_ema else None | |
| os.makedirs(args.outdir, exist_ok=True) | |
| self.ckpt_path = os.path.join(args.outdir, "ckpt_latest.pt") | |
| if os.path.exists(self.ckpt_path): | |
| print(f"[Trainer] Resuming from {self.ckpt_path}") | |
| self.load_ckpt(self.ckpt_path) | |
| def save_ckpt(self): | |
| obj = { | |
| "model": _raw_module(self.model).state_dict(), | |
| "ema": self.ema.shadow if self.ema else None, | |
| "args": vars(self.args), | |
| } | |
| torch.save(obj, self.ckpt_path) | |
| def load_ckpt(self, path: str): | |
| obj = torch.load(path, map_location=self.device) | |
| _raw_module(self.model).load_state_dict(obj["model"]) | |
| if self.ema and obj.get("ema"): | |
| self.ema.shadow = obj["ema"] | |
| def train(self): | |
| self.model.train() | |
| for epoch in range(self.args.epochs): | |
| losses = [] | |
| pbar = tqdm(self.loader, desc=f"Epoch {epoch+1}/{self.args.epochs}") | |
| for x, y in pbar: | |
| x, y = x.to(self.device), y.to(self.device) | |
| t = torch.randint(0, self.args.T, (x.size(0),), device=self.device) | |
| noise = torch.randn_like(x) | |
| a_t = self.diff.sqrt_alphas_cumprod[t][:, None, None, None] | |
| am1_t = self.diff.sqrt_one_minus_alphas_cumprod[t][:, None, None, None] | |
| x_t = a_t * x + am1_t * noise | |
| texts = [CIFAR10_CLASSES[y_i] for y_i in y] | |
| tokens = torch.tensor([self.tokenizer.encode(txt) for txt in texts], device=self.device) | |
| pred = self.model(x_t, t, tokens) | |
| loss = F.mse_loss(pred, noise) | |
| self.opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| nn.utils.clip_grad_norm_(self.model.parameters(), 1.0) | |
| self.opt.step() | |
| if self.ema: | |
| self.ema.update(self.model) | |
| losses.append(loss.item()) | |
| pbar.set_postfix(loss=loss.item()) | |
| avg_loss = sum(losses) / len(losses) | |
| print(f"Epoch {epoch+1}/{self.args.epochs} | Avg Loss: {avg_loss:.4f}") | |
| if (epoch + 1) % max(1, self.args.sample_every) == 0: | |
| self.sample_and_save_grid(epoch) | |
| self.save_ckpt() | |
| def sample_and_save_grid(self, epoch): | |
| self.model.eval() | |
| if self.ema: | |
| shadow = {k: v.clone() for k, v in self.ema.shadow.items()} | |
| bak = {k: v.clone() for k, v in _raw_module(self.model).state_dict().items()} | |
| _raw_module(self.model).load_state_dict(shadow) | |
| prompts = ["dog", "ship on water", "cat", "airplane"] | |
| tokens = torch.tensor([self.tokenizer.encode(p) for p in prompts], device=self.device) | |
| imgs = self.diff.sample((len(prompts), 3, 32, 32), steps=self.args.steps, tokens=tokens) | |
| if self.ema: | |
| _raw_module(self.model).load_state_dict(bak) | |
| imgs = (imgs.clamp(-1, 1) + 1) * 0.5 | |
| grid = vutils.make_grid(imgs, nrow=4) | |
| out_path = os.path.join(self.args.outdir, f"sample_epoch{epoch+1}.png") | |
| vutils.save_image(grid, out_path) | |
| print(f"Saved {out_path}") | |
| def sample(self, prompts=None, steps=50, n_per_prompt=1): | |
| """ | |
| Generate CIFAR-10 32×32 images from text prompts. | |
| Returns tensor in [0,1]. | |
| """ | |
| self.model.eval() | |
| if prompts is None: | |
| prompts = ["cat", "dog"] # fallback | |
| tokens = torch.tensor( | |
| [self.tokenizer.encode(p) for p in prompts for _ in range(n_per_prompt)], | |
| device=self.device | |
| ) | |
| imgs = self.diff.sample( | |
| (len(prompts) * n_per_prompt, 3, 32, 32), | |
| steps=steps, | |
| tokens=tokens | |
| ) | |
| imgs = (imgs.clamp(-1, 1) + 1) * 0.5 | |
| return imgs | |
| # ------------------ CLI ------------------ | |
| def build_parser(): | |
| p = argparse.ArgumentParser(description="Prompt-based CIFAR-10 Diffusion (32×32)") | |
| p.add_argument("--train", action="store_true") | |
| p.add_argument("--sample", action="store_true") | |
| p.add_argument("--data_root", type=str, default="./data") | |
| p.add_argument("--epochs", type=int, default=20) | |
| p.add_argument("--bs", type=int, default=128) | |
| p.add_argument("--lr", type=float, default=1e-4) | |
| p.add_argument("--T", type=int, default=1000) | |
| p.add_argument("--schedule", type=str, default="cosine", choices=["cosine", "linear"]) | |
| p.add_argument("--base", type=int, default=64) | |
| p.add_argument("--cond_dim", type=int, default=256) | |
| p.add_argument("--workers", type=int, default=4) | |
| p.add_argument("--seed", type=int, default=0) | |
| p.add_argument("--outdir", type=str, default="outputs") | |
| p.add_argument("--use_ema", action="store_true") | |
| p.add_argument("--sample_every", type=int, default=5) | |
| p.add_argument("--steps", type=int, default=50) | |
| p.add_argument("--prompts", nargs="*", default=["dog", "cat", "ship", "airplane"]) | |
| return p | |
| Trainer = Trainer | |
| def main(): | |
| args = build_parser().parse_args() | |
| if not args.train and not args.sample: | |
| print("Nothing to do: use --train and/or --sample") | |
| return | |
| if args.train: | |
| tr = Trainer(args) | |
| tr.train() | |
| if args.sample: | |
| tr = Trainer(args) | |
| tr.load_ckpt(tr.ckpt_path) | |
| tokens = torch.tensor([tr.tokenizer.encode(p) for p in args.prompts], device=tr.device) | |
| imgs = tr.diff.sample((len(args.prompts), 3, 32, 32), steps=args.steps, tokens=tokens) | |
| imgs = (imgs.clamp(-1, 1) + 1) * 0.5 | |
| grid = vutils.make_grid(imgs, nrow=4) | |
| os.makedirs(args.outdir, exist_ok=True) | |
| out_path = os.path.join(args.outdir, "samples.png") | |
| vutils.save_image(grid, out_path) | |
| print(f"Saved {out_path}") | |
| if __name__ == "__main__": | |
| main() | |