Parvulus / text_encoder.py
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
2.71 kB
"""Text conditioning encoder for Parvulus."""
from typing import Dict, Optional, Sequence
import torch
import torch.nn as nn
from transformers import AutoTokenizer, T5EncoderModel
DEFAULT_T5_MODEL = "google-t5/t5-small"
DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"
class ParvulusTextEncoder(nn.Module):
"""Mean-pools T5-small token embeddings and projects them to cond_dim."""
def __init__(
self,
cond_dim: int = 256,
model_name: str = DEFAULT_T5_MODEL,
revision: str = DEFAULT_T5_REVISION,
freeze_backbone: bool = True,
):
super().__init__()
self.cond_dim = cond_dim
self.model_name = model_name
self.revision = revision
self.freeze_backbone = freeze_backbone
self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision)
if freeze_backbone:
self.t5.requires_grad_(False)
self.t5.eval()
self.proj = nn.Sequential(
nn.Linear(self.t5.config.d_model, cond_dim),
nn.SiLU(),
nn.Linear(cond_dim, cond_dim),
)
self._embed_cache: Dict[str, torch.Tensor] = {}
def train(self, mode: bool = True):
super().train(mode)
if self.freeze_backbone:
self.t5.eval()
return self
@staticmethod
def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor:
mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype)
summed = (last_hidden * mask).sum(dim=1)
count = mask.sum(dim=1).clamp(min=1e-6)
return summed / count
def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor:
device = next(self.t5.parameters()).device
tokens = self.tokenizer(
list(texts), padding=True, truncation=True, return_tensors="pt"
).to(device)
out = self.t5(**tokens).last_hidden_state
return self._mean_pool(out, tokens["attention_mask"])
def forward(self, texts: Sequence[str]) -> torch.Tensor:
device = next(self.parameters()).device
if self.freeze_backbone:
missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache]
if missing:
with torch.no_grad():
embeds = self._t5_embed(missing)
for t, e in zip(missing, embeds):
self._embed_cache[t] = e.detach()
pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device)
else:
pooled = self._t5_embed(texts).to(device)
return self.proj(pooled)