File size: 2,707 Bytes
67c4f10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
"""Text conditioning encoder for Parvulus."""

from typing import Dict, Optional, Sequence
import torch
import torch.nn as nn
from transformers import AutoTokenizer, T5EncoderModel

DEFAULT_T5_MODEL = "google-t5/t5-small"
DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"


class ParvulusTextEncoder(nn.Module):
    """Mean-pools T5-small token embeddings and projects them to cond_dim."""

    def __init__(
        self,
        cond_dim: int = 256,
        model_name: str = DEFAULT_T5_MODEL,
        revision: str = DEFAULT_T5_REVISION,
        freeze_backbone: bool = True,
    ):
        super().__init__()
        self.cond_dim = cond_dim
        self.model_name = model_name
        self.revision = revision
        self.freeze_backbone = freeze_backbone

        self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
        self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision)
        if freeze_backbone:
            self.t5.requires_grad_(False)
            self.t5.eval()

        self.proj = nn.Sequential(
            nn.Linear(self.t5.config.d_model, cond_dim),
            nn.SiLU(),
            nn.Linear(cond_dim, cond_dim),
        )
        self._embed_cache: Dict[str, torch.Tensor] = {}

    def train(self, mode: bool = True):
        super().train(mode)
        if self.freeze_backbone:
            self.t5.eval()
        return self

    @staticmethod
    def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor:
        mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype)
        summed = (last_hidden * mask).sum(dim=1)
        count = mask.sum(dim=1).clamp(min=1e-6)
        return summed / count

    def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor:
        device = next(self.t5.parameters()).device
        tokens = self.tokenizer(
            list(texts), padding=True, truncation=True, return_tensors="pt"
        ).to(device)
        out = self.t5(**tokens).last_hidden_state
        return self._mean_pool(out, tokens["attention_mask"])

    def forward(self, texts: Sequence[str]) -> torch.Tensor:
        device = next(self.parameters()).device
        if self.freeze_backbone:
            missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache]
            if missing:
                with torch.no_grad():
                    embeds = self._t5_embed(missing)
                for t, e in zip(missing, embeds):
                    self._embed_cache[t] = e.detach()
            pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device)
        else:
            pooled = self._t5_embed(texts).to(device)
        return self.proj(pooled)