Spaces:
Sleeping
Sleeping
File size: 1,875 Bytes
c388140 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | from __future__ import annotations
from dataclasses import dataclass
from typing import Optional
import torch
from torch import nn
@dataclass(frozen=True)
class TextEncoderConfig:
"""
文本编码器配置:
- 目标是轻量、可训练、可随 checkpoint 一起保存,不依赖外部大模型。
"""
vocab_size: int
max_len: int = 32
emb_dim: int = 128
out_dim: int = 256
dropout: float = 0.0
class TextEncoder(nn.Module):
"""
极简 TextEncoder:
- token embedding
- masked mean pooling
- MLP 投影到 out_dim(通常与 UNet time_emb_dim 对齐)
"""
def __init__(self, cfg: TextEncoderConfig, pad_id: int = 0):
super().__init__()
self.cfg = cfg
self.pad_id = int(pad_id)
self.emb = nn.Embedding(cfg.vocab_size, cfg.emb_dim, padding_idx=self.pad_id)
self.proj = nn.Sequential(
nn.Dropout(cfg.dropout) if cfg.dropout > 0 else nn.Identity(),
nn.Linear(cfg.emb_dim, cfg.out_dim),
nn.SiLU(),
nn.Linear(cfg.out_dim, cfg.out_dim),
)
def forward(self, tokens: torch.Tensor, mask: Optional[torch.Tensor] = None) -> torch.Tensor:
"""
tokens: (B, L) int64
mask: (B, L) bool/0-1;为 None 时自动根据 pad_id 生成
返回:
- cond: (B, out_dim)
"""
if mask is None:
mask = (tokens != self.pad_id)
mask_f = mask.to(dtype=self.emb.weight.dtype)
x = self.emb(tokens) # (B,L,D)
# masked mean
denom = mask_f.sum(dim=1, keepdim=True).clamp(min=1.0)
pooled = (x * mask_f.unsqueeze(-1)).sum(dim=1) / denom
# 若整句为空(mask 全 0),pooled 会是全 0,等价于无条件
return self.proj(pooled)
|