Download text_encoder.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 2.71 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/text_encoder.py
- Command line
-
hf download hf://aasivas/Parvulus/text_encoder.py
-
curl -L -o text_encoder.py https://huggingface.co/aasivas/Parvulus/resolve/main/text_encoder.py
2.71 kB
| """Text conditioning encoder for Parvulus.""" | |
| from typing import Dict, Optional, Sequence | |
| import torch | |
| import torch.nn as nn | |
| from transformers import AutoTokenizer, T5EncoderModel | |
| DEFAULT_T5_MODEL = "google-t5/t5-small" | |
| DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4" | |
| class ParvulusTextEncoder(nn.Module): | |
| """Mean-pools T5-small token embeddings and projects them to cond_dim.""" | |
| def __init__( | |
| self, | |
| cond_dim: int = 256, | |
| model_name: str = DEFAULT_T5_MODEL, | |
| revision: str = DEFAULT_T5_REVISION, | |
| freeze_backbone: bool = True, | |
| ): | |
| super().__init__() | |
| self.cond_dim = cond_dim | |
| self.model_name = model_name | |
| self.revision = revision | |
| self.freeze_backbone = freeze_backbone | |
| self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision) | |
| self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision) | |
| if freeze_backbone: | |
| self.t5.requires_grad_(False) | |
| self.t5.eval() | |
| self.proj = nn.Sequential( | |
| nn.Linear(self.t5.config.d_model, cond_dim), | |
| nn.SiLU(), | |
| nn.Linear(cond_dim, cond_dim), | |
| ) | |
| self._embed_cache: Dict[str, torch.Tensor] = {} | |
| def train(self, mode: bool = True): | |
| super().train(mode) | |
| if self.freeze_backbone: | |
| self.t5.eval() | |
| return self | |
| def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor: | |
| mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype) | |
| summed = (last_hidden * mask).sum(dim=1) | |
| count = mask.sum(dim=1).clamp(min=1e-6) | |
| return summed / count | |
| def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor: | |
| device = next(self.t5.parameters()).device | |
| tokens = self.tokenizer( | |
| list(texts), padding=True, truncation=True, return_tensors="pt" | |
| ).to(device) | |
| out = self.t5(**tokens).last_hidden_state | |
| return self._mean_pool(out, tokens["attention_mask"]) | |
| def forward(self, texts: Sequence[str]) -> torch.Tensor: | |
| device = next(self.parameters()).device | |
| if self.freeze_backbone: | |
| missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache] | |
| if missing: | |
| with torch.no_grad(): | |
| embeds = self._t5_embed(missing) | |
| for t, e in zip(missing, embeds): | |
| self._embed_cache[t] = e.detach() | |
| pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device) | |
| else: | |
| pooled = self._t5_embed(texts).to(device) | |
| return self.proj(pooled) | |