import torch import torch.nn as nn class MultimodalConcat_TemporalTF(nn.Module): """Weekly tokens = concat(text_emb_w, image_emb_w, has_image_w, image_age_w) -> causal temporal transformer. Horizon embedding, week signal + learned positions, causal TransformerEncoder, last-token pooling and a small regression head. """ def __init__( self, text_dim: int = 1024, image_dim: int = 768, d_model: int = 64, nhead: int = 4, num_layers: int = 2, dim_ff: int = 128, dropout: float = 0.3, input_dropout: float = 0.2, max_weeks: int = 22, pool: str = "last", horizon_emb_dim: int = 16, ): super().__init__() assert pool in ["last", "mean"] self.pool = pool self.max_weeks = max_weeks in_dim = text_dim + image_dim + 2 self.input_proj = nn.Sequential( nn.Dropout(input_dropout), nn.Linear(in_dim, d_model), nn.GELU(), nn.Dropout(dropout), nn.Linear(d_model, d_model), ) self.in_ln = nn.LayerNorm(d_model) self.horizon_emb = nn.Embedding(max_weeks + 1, horizon_emb_dim) self.horizon_proj = nn.Linear(horizon_emb_dim, d_model) self.week_proj = nn.Linear(1, d_model) self.pos_emb = nn.Parameter(torch.zeros(1, max_weeks + 1, d_model)) nn.init.trunc_normal_(self.pos_emb, std=0.02) enc_layer = nn.TransformerEncoderLayer( d_model=d_model, nhead=nhead, dim_feedforward=dim_ff, dropout=dropout, activation="gelu", batch_first=True, norm_first=True, ) self.temporal_tf = nn.TransformerEncoder(enc_layer, num_layers=num_layers, enable_nested_tensor=False) self.head = nn.Sequential( nn.LayerNorm(d_model + horizon_emb_dim), nn.Linear(d_model + horizon_emb_dim, d_model), nn.GELU(), nn.Dropout(dropout), nn.Linear(d_model, 1), ) def forward(self, text, image, img_mask, img_age, horizon_idx=None, causal=True): m = img_mask.unsqueeze(-1).to(image.dtype) age = img_age.unsqueeze(-1).to(image.dtype) x = torch.cat([text, image * m, m, age * m], dim=-1) B, t, _ = x.shape device = x.device if horizon_idx is None: horizon_idx = t if not torch.is_tensor(horizon_idx): horizon_idx = torch.tensor(horizon_idx, device=device) h_idx = horizon_idx.long().expand(B).clamp(min=1, max=self.max_weeks) h_emb = self.horizon_emb(h_idx) h_tok = self.horizon_proj(h_emb) tok = self.in_ln(self.input_proj(x)) + h_tok[:, None, :] weeks = torch.arange(1, t + 1, device=device).float() / self.max_weeks tok = tok + self.week_proj(weeks[None, :, None].expand(B, -1, -1)) + self.pos_emb[:, :t, :] attn_mask = None if causal: attn_mask = torch.triu(torch.ones(t, t, device=device, dtype=torch.bool), diagonal=1) h = self.temporal_tf(tok, mask=attn_mask) pooled = h[:, -1, :] if self.pool == "last" else h.mean(dim=1) return self.head(torch.cat([pooled, h_emb], dim=-1)).squeeze(-1)