File size: 4,638 Bytes
c388140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
from __future__ import annotations

from dataclasses import dataclass
from typing import List, Optional, Tuple

from PIL import Image
import torch
from torch.utils.data import Dataset
from torchvision import transforms as T

from blockdiffusion.data.captions import CaptionItem
from blockdiffusion.text.simple_tokenizer import SimpleVocab


class TextureCaptionDataset(Dataset):
    """
    图文配对纹理数据集:
    - 输入:CaptionItem 列表(包含 image_path 与 text)
    - 输出:(image[-1,1], tokens, mask)
    """

    def __init__(
        self,
        items: List[CaptionItem],
        vocab: SimpleVocab,
        image_size: int = 16,
        channels: int = 3,
        max_text_len: int = 32,
        random_flip: bool = True,
        pretokenize: bool = False,
        append_alpha_tag: bool = False,
        alpha_tag_transparent: str = "transparent",
        alpha_tag_opaque: str = "opaque",
        verbose: bool = False,
        log_every: int = 50_000,
    ):
        if len(items) == 0:
            raise ValueError("items 不能为空")
        self.items = items
        self.vocab = vocab
        self.image_size = int(image_size)
        self.channels = int(channels)
        if self.channels not in (3, 4):
            raise ValueError("channels 仅支持 3(RGB) 或 4(RGBA)")
        self.max_text_len = int(max_text_len)
        self.pretokenize = bool(pretokenize)
        self.append_alpha_tag = bool(append_alpha_tag)
        self.alpha_tag_transparent = str(alpha_tag_transparent or "").strip()
        self.alpha_tag_opaque = str(alpha_tag_opaque or "").strip()
        self.verbose = bool(verbose)
        self.log_every = int(log_every)

        # 透明/不透明标签需要在 __getitem__ 里基于图像 alpha 判定;
        # pretokenize 不读取图像,因此两者不兼容。
        if self.append_alpha_tag and self.pretokenize:
            raise ValueError("append_alpha_tag 与 pretokenize 不兼容:请关闭 pretokenize")

        tfms = [
            T.Resize((self.image_size, self.image_size), interpolation=T.InterpolationMode.NEAREST),
        ]
        if random_flip:
            tfms.append(T.RandomHorizontalFlip(p=0.5))
        tfms.extend(
            [
                T.ToTensor(),
                # 用 Normalize 替代 lambda,保证 Windows 多进程 DataLoader 可 pickle
                T.Normalize(mean=[0.5] * self.channels, std=[0.5] * self.channels),
            ]
        )
        self.transform = T.Compose(tfms)

        # 可选:预编码 tokens/mask(大数据集会耗时/占内存,默认关闭)
        self._tokens = None
        self._mask = None
        if self.pretokenize:
            tokens_list: List[torch.Tensor] = []
            mask_list: List[torch.Tensor] = []
            for i, it in enumerate(items, start=1):
                ids = vocab.encode(it.text, max_len=self.max_text_len)
                tok = torch.tensor(ids, dtype=torch.long)
                m = (tok != vocab.pad_id)
                tokens_list.append(tok)
                mask_list.append(m)
                if self.verbose and self.log_every > 0 and (i % self.log_every == 0):
                    print(f"[pretok] 已预编码 {i} 条文本...")
            self._tokens = tokens_list
            self._mask = mask_list
            if self.verbose:
                print(f"[pretok] 完成:共 {len(items)} 条文本")

    def __len__(self) -> int:
        return len(self.items)

    def __getitem__(self, idx: int):
        it = self.items[idx]
        with Image.open(it.image_path) as im:
            im = im.convert("RGBA" if self.channels == 4 else "RGB")
            x = self.transform(im)
        if self._tokens is not None and self._mask is not None:
            return x, self._tokens[idx], self._mask[idx]
        # 未预编码:按需编码(更快启动、更省内存)
        text = it.text
        # 可选:自动追加透明/不透明标签,让模型学会“透明度受提示词影响”
        if self.append_alpha_tag and self.channels == 4:
            # im 已被 convert("RGBA")
            alpha = im.getchannel("A")
            mn, mx = alpha.getextrema()
            # 只要出现 <255 就认为“含透明”
            tag = self.alpha_tag_transparent if int(mn) < 255 else self.alpha_tag_opaque
            tag = (tag or "").strip()
            if tag:
                text = f"{text} {tag}".strip() if text else tag

        ids = self.vocab.encode(text, max_len=self.max_text_len)
        tok = torch.tensor(ids, dtype=torch.long)
        m = (tok != self.vocab.pad_id)
        return x, tok, m