File size: 1,875 Bytes
c388140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
from __future__ import annotations

from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn


@dataclass(frozen=True)
class TextEncoderConfig:
    """

    文本编码器配置:

    - 目标是轻量、可训练、可随 checkpoint 一起保存,不依赖外部大模型。

    """

    vocab_size: int
    max_len: int = 32
    emb_dim: int = 128
    out_dim: int = 256
    dropout: float = 0.0


class TextEncoder(nn.Module):
    """

    极简 TextEncoder:

    - token embedding

    - masked mean pooling

    - MLP 投影到 out_dim(通常与 UNet time_emb_dim 对齐)

    """

    def __init__(self, cfg: TextEncoderConfig, pad_id: int = 0):
        super().__init__()
        self.cfg = cfg
        self.pad_id = int(pad_id)

        self.emb = nn.Embedding(cfg.vocab_size, cfg.emb_dim, padding_idx=self.pad_id)
        self.proj = nn.Sequential(
            nn.Dropout(cfg.dropout) if cfg.dropout > 0 else nn.Identity(),
            nn.Linear(cfg.emb_dim, cfg.out_dim),
            nn.SiLU(),
            nn.Linear(cfg.out_dim, cfg.out_dim),
        )

    def forward(self, tokens: torch.Tensor, mask: Optional[torch.Tensor] = None) -> torch.Tensor:
        """

        tokens: (B, L) int64

        mask:   (B, L) bool/0-1;为 None 时自动根据 pad_id 生成

        返回:

        - cond: (B, out_dim)

        """
        if mask is None:
            mask = (tokens != self.pad_id)
        mask_f = mask.to(dtype=self.emb.weight.dtype)

        x = self.emb(tokens)  # (B,L,D)
        # masked mean
        denom = mask_f.sum(dim=1, keepdim=True).clamp(min=1.0)
        pooled = (x * mask_f.unsqueeze(-1)).sum(dim=1) / denom

        # 若整句为空(mask 全 0),pooled 会是全 0,等价于无条件
        return self.proj(pooled)