multimodal-yield-estimation / hf /modeling_multimodal.py
Sarikaa-Sridhar's picture
multi-modal yield estimation code base
4191737 verified
Raw History Blame Contribute Delete
3.09 kB
from dataclasses import dataclass
import torch
from torch import nn
from transformers import PreTrainedModel
from transformers.modeling_outputs import ModelOutput
from .configuration_multimodal import MultimodalYieldConfig
from .multimodal_concat_tf import MultimodalConcat_TemporalTF
@dataclass
class MultimodalYieldOutput(ModelOutput):
loss: torch.Tensor | None = None
logits: torch.Tensor | None = None
predictions: torch.Tensor | None = None
def _stat(mean, std, dim):
if mean is None:
return torch.zeros(dim), torch.ones(dim)
return torch.tensor(mean, dtype=torch.float32), torch.tensor(std, dtype=torch.float32)
class MultimodalYieldForRegression(PreTrainedModel):
"""Inputs (raw, un-normalized):
text [B, T, 1024] weekly Qwen3 caption embeddings
image [B, T, 768] DINOv3 feature of latest image at/before each week (any value where img_mask=0)
img_mask [B, T] 1 if an image is available at/before that week
img_age [B, T] weeks since that image was taken (0 = new image this week)
cutoff int use only the first `cutoff` weeks after planting (defaults to T)
"""
config_class = MultimodalYieldConfig
base_model_prefix = "yield_model"
def __init__(self, config: MultimodalYieldConfig):
super().__init__(config)
tm, ts = _stat(config.text_mean, config.text_std, config.text_dim)
im, is_ = _stat(config.image_mean, config.image_std, config.image_dim)
self.register_buffer("text_mean", tm)
self.register_buffer("text_std", ts)
self.register_buffer("image_mean", im)
self.register_buffer("image_std", is_)
self.yield_model = MultimodalConcat_TemporalTF(
text_dim=config.text_dim,
image_dim=config.image_dim,
d_model=config.d_model,
nhead=config.nhead,
num_layers=config.num_layers,
dim_ff=config.dim_ff,
dropout=config.dropout,
input_dropout=config.input_dropout,
max_weeks=config.K,
pool=config.pool,
)
self.post_init()
def forward(self, text, image, img_mask, img_age, cutoff=None, labels=None, return_dict=True):
T = text.shape[1]
t = T if cutoff is None else min(int(cutoff), T)
txt = (text[:, :t].float() - self.text_mean) / self.text_std
img = (image[:, :t].float() - self.image_mean) / self.image_std
msk = img_mask[:, :t].float()
age = img_age[:, :t].float() / self.config.K # weeks -> ~[0, 1]
logits = self.yield_model(text=txt, image=img, img_mask=msk, img_age=age, horizon_idx=t, causal=True)
predictions = logits * self.config.y_std + self.config.y_mean
loss = None
if labels is not None:
loss = nn.functional.mse_loss(logits, (labels - self.config.y_mean) / self.config.y_std)
if not return_dict:
return (loss, logits, predictions)
return MultimodalYieldOutput(loss=loss, logits=logits, predictions=predictions)