from dataclasses import dataclass import torch from torch import nn from transformers import PreTrainedModel from transformers.modeling_outputs import ModelOutput from .configuration_multimodal import MultimodalYieldConfig from .multimodal_concat_tf import MultimodalConcat_TemporalTF @dataclass class MultimodalYieldOutput(ModelOutput): loss: torch.Tensor | None = None logits: torch.Tensor | None = None predictions: torch.Tensor | None = None def _stat(mean, std, dim): if mean is None: return torch.zeros(dim), torch.ones(dim) return torch.tensor(mean, dtype=torch.float32), torch.tensor(std, dtype=torch.float32) class MultimodalYieldForRegression(PreTrainedModel): """Inputs (raw, un-normalized): text [B, T, 1024] weekly Qwen3 caption embeddings image [B, T, 768] DINOv3 feature of latest image at/before each week (any value where img_mask=0) img_mask [B, T] 1 if an image is available at/before that week img_age [B, T] weeks since that image was taken (0 = new image this week) cutoff int use only the first `cutoff` weeks after planting (defaults to T) """ config_class = MultimodalYieldConfig base_model_prefix = "yield_model" def __init__(self, config: MultimodalYieldConfig): super().__init__(config) tm, ts = _stat(config.text_mean, config.text_std, config.text_dim) im, is_ = _stat(config.image_mean, config.image_std, config.image_dim) self.register_buffer("text_mean", tm) self.register_buffer("text_std", ts) self.register_buffer("image_mean", im) self.register_buffer("image_std", is_) self.yield_model = MultimodalConcat_TemporalTF( text_dim=config.text_dim, image_dim=config.image_dim, d_model=config.d_model, nhead=config.nhead, num_layers=config.num_layers, dim_ff=config.dim_ff, dropout=config.dropout, input_dropout=config.input_dropout, max_weeks=config.K, pool=config.pool, ) self.post_init() def forward(self, text, image, img_mask, img_age, cutoff=None, labels=None, return_dict=True): T = text.shape[1] t = T if cutoff is None else min(int(cutoff), T) txt = (text[:, :t].float() - self.text_mean) / self.text_std img = (image[:, :t].float() - self.image_mean) / self.image_std msk = img_mask[:, :t].float() age = img_age[:, :t].float() / self.config.K # weeks -> ~[0, 1] logits = self.yield_model(text=txt, image=img, img_mask=msk, img_age=age, horizon_idx=t, causal=True) predictions = logits * self.config.y_std + self.config.y_mean loss = None if labels is not None: loss = nn.functional.mse_loss(logits, (labels - self.config.y_mean) / self.config.y_std) if not return_dict: return (loss, logits, predictions) return MultimodalYieldOutput(loss=loss, logits=logits, predictions=predictions)