"""Text conditioning encoder for Parvulus.""" from typing import Dict, Optional, Sequence import torch import torch.nn as nn from transformers import AutoTokenizer, T5EncoderModel DEFAULT_T5_MODEL = "google-t5/t5-small" DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4" class ParvulusTextEncoder(nn.Module): """Mean-pools T5-small token embeddings and projects them to cond_dim.""" def __init__( self, cond_dim: int = 256, model_name: str = DEFAULT_T5_MODEL, revision: str = DEFAULT_T5_REVISION, freeze_backbone: bool = True, ): super().__init__() self.cond_dim = cond_dim self.model_name = model_name self.revision = revision self.freeze_backbone = freeze_backbone self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision) self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision) if freeze_backbone: self.t5.requires_grad_(False) self.t5.eval() self.proj = nn.Sequential( nn.Linear(self.t5.config.d_model, cond_dim), nn.SiLU(), nn.Linear(cond_dim, cond_dim), ) self._embed_cache: Dict[str, torch.Tensor] = {} def train(self, mode: bool = True): super().train(mode) if self.freeze_backbone: self.t5.eval() return self @staticmethod def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor: mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype) summed = (last_hidden * mask).sum(dim=1) count = mask.sum(dim=1).clamp(min=1e-6) return summed / count def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor: device = next(self.t5.parameters()).device tokens = self.tokenizer( list(texts), padding=True, truncation=True, return_tensors="pt" ).to(device) out = self.t5(**tokens).last_hidden_state return self._mean_pool(out, tokens["attention_mask"]) def forward(self, texts: Sequence[str]) -> torch.Tensor: device = next(self.parameters()).device if self.freeze_backbone: missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache] if missing: with torch.no_grad(): embeds = self._t5_embed(missing) for t, e in zip(missing, embeds): self._embed_cache[t] = e.detach() pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device) else: pooled = self._t5_embed(texts).to(device) return self.proj(pooled)