multimodal-yield-estimation / multimodal_concat_tf.py
Sarikaa-Sridhar's picture
multi-modal yield estimation code base
4191737 verified
Raw History Blame Contribute Delete
3.25 kB
import torch
import torch.nn as nn
class MultimodalConcat_TemporalTF(nn.Module):
"""Weekly tokens = concat(text_emb_w, image_emb_w, has_image_w, image_age_w) -> causal temporal transformer.
Horizon embedding, week signal + learned positions, causal TransformerEncoder,
last-token pooling and a small regression head.
"""
def __init__(
self,
text_dim: int = 1024,
image_dim: int = 768,
d_model: int = 64,
nhead: int = 4,
num_layers: int = 2,
dim_ff: int = 128,
dropout: float = 0.3,
input_dropout: float = 0.2,
max_weeks: int = 22,
pool: str = "last",
horizon_emb_dim: int = 16,
):
super().__init__()
assert pool in ["last", "mean"]
self.pool = pool
self.max_weeks = max_weeks
in_dim = text_dim + image_dim + 2
self.input_proj = nn.Sequential(
nn.Dropout(input_dropout),
nn.Linear(in_dim, d_model),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(d_model, d_model),
)
self.in_ln = nn.LayerNorm(d_model)
self.horizon_emb = nn.Embedding(max_weeks + 1, horizon_emb_dim)
self.horizon_proj = nn.Linear(horizon_emb_dim, d_model)
self.week_proj = nn.Linear(1, d_model)
self.pos_emb = nn.Parameter(torch.zeros(1, max_weeks + 1, d_model))
nn.init.trunc_normal_(self.pos_emb, std=0.02)
enc_layer = nn.TransformerEncoderLayer(
d_model=d_model, nhead=nhead, dim_feedforward=dim_ff,
dropout=dropout, activation="gelu", batch_first=True, norm_first=True,
)
self.temporal_tf = nn.TransformerEncoder(enc_layer, num_layers=num_layers, enable_nested_tensor=False)
self.head = nn.Sequential(
nn.LayerNorm(d_model + horizon_emb_dim),
nn.Linear(d_model + horizon_emb_dim, d_model),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(d_model, 1),
)
def forward(self, text, image, img_mask, img_age, horizon_idx=None, causal=True):
m = img_mask.unsqueeze(-1).to(image.dtype)
age = img_age.unsqueeze(-1).to(image.dtype)
x = torch.cat([text, image * m, m, age * m], dim=-1)
B, t, _ = x.shape
device = x.device
if horizon_idx is None:
horizon_idx = t
if not torch.is_tensor(horizon_idx):
horizon_idx = torch.tensor(horizon_idx, device=device)
h_idx = horizon_idx.long().expand(B).clamp(min=1, max=self.max_weeks)
h_emb = self.horizon_emb(h_idx)
h_tok = self.horizon_proj(h_emb)
tok = self.in_ln(self.input_proj(x)) + h_tok[:, None, :]
weeks = torch.arange(1, t + 1, device=device).float() / self.max_weeks
tok = tok + self.week_proj(weeks[None, :, None].expand(B, -1, -1)) + self.pos_emb[:, :t, :]
attn_mask = None
if causal:
attn_mask = torch.triu(torch.ones(t, t, device=device, dtype=torch.bool), diagonal=1)
h = self.temporal_tf(tok, mask=attn_mask)
pooled = h[:, -1, :] if self.pool == "last" else h.mean(dim=1)
return self.head(torch.cat([pooled, h_emb], dim=-1)).squeeze(-1)