from __future__ import annotations from dataclasses import dataclass from typing import List, Optional, Tuple from PIL import Image import torch from torch.utils.data import Dataset from torchvision import transforms as T from blockdiffusion.data.captions import CaptionItem from blockdiffusion.text.simple_tokenizer import SimpleVocab class TextureCaptionDataset(Dataset): """ 图文配对纹理数据集: - 输入:CaptionItem 列表(包含 image_path 与 text) - 输出:(image[-1,1], tokens, mask) """ def __init__( self, items: List[CaptionItem], vocab: SimpleVocab, image_size: int = 16, channels: int = 3, max_text_len: int = 32, random_flip: bool = True, pretokenize: bool = False, append_alpha_tag: bool = False, alpha_tag_transparent: str = "transparent", alpha_tag_opaque: str = "opaque", verbose: bool = False, log_every: int = 50_000, ): if len(items) == 0: raise ValueError("items 不能为空") self.items = items self.vocab = vocab self.image_size = int(image_size) self.channels = int(channels) if self.channels not in (3, 4): raise ValueError("channels 仅支持 3(RGB) 或 4(RGBA)") self.max_text_len = int(max_text_len) self.pretokenize = bool(pretokenize) self.append_alpha_tag = bool(append_alpha_tag) self.alpha_tag_transparent = str(alpha_tag_transparent or "").strip() self.alpha_tag_opaque = str(alpha_tag_opaque or "").strip() self.verbose = bool(verbose) self.log_every = int(log_every) # 透明/不透明标签需要在 __getitem__ 里基于图像 alpha 判定; # pretokenize 不读取图像,因此两者不兼容。 if self.append_alpha_tag and self.pretokenize: raise ValueError("append_alpha_tag 与 pretokenize 不兼容:请关闭 pretokenize") tfms = [ T.Resize((self.image_size, self.image_size), interpolation=T.InterpolationMode.NEAREST), ] if random_flip: tfms.append(T.RandomHorizontalFlip(p=0.5)) tfms.extend( [ T.ToTensor(), # 用 Normalize 替代 lambda,保证 Windows 多进程 DataLoader 可 pickle T.Normalize(mean=[0.5] * self.channels, std=[0.5] * self.channels), ] ) self.transform = T.Compose(tfms) # 可选:预编码 tokens/mask(大数据集会耗时/占内存,默认关闭) self._tokens = None self._mask = None if self.pretokenize: tokens_list: List[torch.Tensor] = [] mask_list: List[torch.Tensor] = [] for i, it in enumerate(items, start=1): ids = vocab.encode(it.text, max_len=self.max_text_len) tok = torch.tensor(ids, dtype=torch.long) m = (tok != vocab.pad_id) tokens_list.append(tok) mask_list.append(m) if self.verbose and self.log_every > 0 and (i % self.log_every == 0): print(f"[pretok] 已预编码 {i} 条文本...") self._tokens = tokens_list self._mask = mask_list if self.verbose: print(f"[pretok] 完成:共 {len(items)} 条文本") def __len__(self) -> int: return len(self.items) def __getitem__(self, idx: int): it = self.items[idx] with Image.open(it.image_path) as im: im = im.convert("RGBA" if self.channels == 4 else "RGB") x = self.transform(im) if self._tokens is not None and self._mask is not None: return x, self._tokens[idx], self._mask[idx] # 未预编码:按需编码(更快启动、更省内存) text = it.text # 可选:自动追加透明/不透明标签,让模型学会“透明度受提示词影响” if self.append_alpha_tag and self.channels == 4: # im 已被 convert("RGBA") alpha = im.getchannel("A") mn, mx = alpha.getextrema() # 只要出现 <255 就认为“含透明” tag = self.alpha_tag_transparent if int(mn) < 255 else self.alpha_tag_opaque tag = (tag or "").strip() if tag: text = f"{text} {tag}".strip() if text else tag ids = self.vocab.encode(text, max_len=self.max_text_len) tok = torch.tensor(ids, dtype=torch.long) m = (tok != self.vocab.pad_id) return x, tok, m