Spaces:
Sleeping
Sleeping
File size: 4,638 Bytes
c388140 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | from __future__ import annotations
from dataclasses import dataclass
from typing import List, Optional, Tuple
from PIL import Image
import torch
from torch.utils.data import Dataset
from torchvision import transforms as T
from blockdiffusion.data.captions import CaptionItem
from blockdiffusion.text.simple_tokenizer import SimpleVocab
class TextureCaptionDataset(Dataset):
"""
图文配对纹理数据集:
- 输入:CaptionItem 列表(包含 image_path 与 text)
- 输出:(image[-1,1], tokens, mask)
"""
def __init__(
self,
items: List[CaptionItem],
vocab: SimpleVocab,
image_size: int = 16,
channels: int = 3,
max_text_len: int = 32,
random_flip: bool = True,
pretokenize: bool = False,
append_alpha_tag: bool = False,
alpha_tag_transparent: str = "transparent",
alpha_tag_opaque: str = "opaque",
verbose: bool = False,
log_every: int = 50_000,
):
if len(items) == 0:
raise ValueError("items 不能为空")
self.items = items
self.vocab = vocab
self.image_size = int(image_size)
self.channels = int(channels)
if self.channels not in (3, 4):
raise ValueError("channels 仅支持 3(RGB) 或 4(RGBA)")
self.max_text_len = int(max_text_len)
self.pretokenize = bool(pretokenize)
self.append_alpha_tag = bool(append_alpha_tag)
self.alpha_tag_transparent = str(alpha_tag_transparent or "").strip()
self.alpha_tag_opaque = str(alpha_tag_opaque or "").strip()
self.verbose = bool(verbose)
self.log_every = int(log_every)
# 透明/不透明标签需要在 __getitem__ 里基于图像 alpha 判定;
# pretokenize 不读取图像,因此两者不兼容。
if self.append_alpha_tag and self.pretokenize:
raise ValueError("append_alpha_tag 与 pretokenize 不兼容:请关闭 pretokenize")
tfms = [
T.Resize((self.image_size, self.image_size), interpolation=T.InterpolationMode.NEAREST),
]
if random_flip:
tfms.append(T.RandomHorizontalFlip(p=0.5))
tfms.extend(
[
T.ToTensor(),
# 用 Normalize 替代 lambda,保证 Windows 多进程 DataLoader 可 pickle
T.Normalize(mean=[0.5] * self.channels, std=[0.5] * self.channels),
]
)
self.transform = T.Compose(tfms)
# 可选:预编码 tokens/mask(大数据集会耗时/占内存,默认关闭)
self._tokens = None
self._mask = None
if self.pretokenize:
tokens_list: List[torch.Tensor] = []
mask_list: List[torch.Tensor] = []
for i, it in enumerate(items, start=1):
ids = vocab.encode(it.text, max_len=self.max_text_len)
tok = torch.tensor(ids, dtype=torch.long)
m = (tok != vocab.pad_id)
tokens_list.append(tok)
mask_list.append(m)
if self.verbose and self.log_every > 0 and (i % self.log_every == 0):
print(f"[pretok] 已预编码 {i} 条文本...")
self._tokens = tokens_list
self._mask = mask_list
if self.verbose:
print(f"[pretok] 完成:共 {len(items)} 条文本")
def __len__(self) -> int:
return len(self.items)
def __getitem__(self, idx: int):
it = self.items[idx]
with Image.open(it.image_path) as im:
im = im.convert("RGBA" if self.channels == 4 else "RGB")
x = self.transform(im)
if self._tokens is not None and self._mask is not None:
return x, self._tokens[idx], self._mask[idx]
# 未预编码:按需编码(更快启动、更省内存)
text = it.text
# 可选:自动追加透明/不透明标签,让模型学会“透明度受提示词影响”
if self.append_alpha_tag and self.channels == 4:
# im 已被 convert("RGBA")
alpha = im.getchannel("A")
mn, mx = alpha.getextrema()
# 只要出现 <255 就认为“含透明”
tag = self.alpha_tag_transparent if int(mn) < 255 else self.alpha_tag_opaque
tag = (tag or "").strip()
if tag:
text = f"{text} {tag}".strip() if text else tag
ids = self.vocab.encode(text, max_len=self.max_text_len)
tok = torch.tensor(ids, dtype=torch.long)
m = (tok != self.vocab.pad_id)
return x, tok, m
|