File size: 2,707 Bytes
67c4f10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | """Text conditioning encoder for Parvulus."""
from typing import Dict, Optional, Sequence
import torch
import torch.nn as nn
from transformers import AutoTokenizer, T5EncoderModel
DEFAULT_T5_MODEL = "google-t5/t5-small"
DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"
class ParvulusTextEncoder(nn.Module):
"""Mean-pools T5-small token embeddings and projects them to cond_dim."""
def __init__(
self,
cond_dim: int = 256,
model_name: str = DEFAULT_T5_MODEL,
revision: str = DEFAULT_T5_REVISION,
freeze_backbone: bool = True,
):
super().__init__()
self.cond_dim = cond_dim
self.model_name = model_name
self.revision = revision
self.freeze_backbone = freeze_backbone
self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision)
if freeze_backbone:
self.t5.requires_grad_(False)
self.t5.eval()
self.proj = nn.Sequential(
nn.Linear(self.t5.config.d_model, cond_dim),
nn.SiLU(),
nn.Linear(cond_dim, cond_dim),
)
self._embed_cache: Dict[str, torch.Tensor] = {}
def train(self, mode: bool = True):
super().train(mode)
if self.freeze_backbone:
self.t5.eval()
return self
@staticmethod
def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor:
mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype)
summed = (last_hidden * mask).sum(dim=1)
count = mask.sum(dim=1).clamp(min=1e-6)
return summed / count
def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor:
device = next(self.t5.parameters()).device
tokens = self.tokenizer(
list(texts), padding=True, truncation=True, return_tensors="pt"
).to(device)
out = self.t5(**tokens).last_hidden_state
return self._mean_pool(out, tokens["attention_mask"])
def forward(self, texts: Sequence[str]) -> torch.Tensor:
device = next(self.parameters()).device
if self.freeze_backbone:
missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache]
if missing:
with torch.no_grad():
embeds = self._t5_embed(missing)
for t, e in zip(missing, embeds):
self._embed_cache[t] = e.detach()
pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device)
else:
pooled = self._t5_embed(texts).to(device)
return self.proj(pooled)
|