File size: 3,092 Bytes
4191737
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
from dataclasses import dataclass

import torch
from torch import nn
from transformers import PreTrainedModel
from transformers.modeling_outputs import ModelOutput

from .configuration_multimodal import MultimodalYieldConfig
from .multimodal_concat_tf import MultimodalConcat_TemporalTF


@dataclass
class MultimodalYieldOutput(ModelOutput):
    loss: torch.Tensor | None = None
    logits: torch.Tensor | None = None
    predictions: torch.Tensor | None = None


def _stat(mean, std, dim):
    if mean is None:
        return torch.zeros(dim), torch.ones(dim)
    return torch.tensor(mean, dtype=torch.float32), torch.tensor(std, dtype=torch.float32)


class MultimodalYieldForRegression(PreTrainedModel):
    """Inputs (raw, un-normalized):
        text     [B, T, 1024]  weekly Qwen3 caption embeddings
        image    [B, T, 768]   DINOv3 feature of latest image at/before each week (any value where img_mask=0)
        img_mask [B, T]        1 if an image is available at/before that week
        img_age  [B, T]        weeks since that image was taken (0 = new image this week)
        cutoff   int           use only the first `cutoff` weeks after planting (defaults to T)
    """

    config_class = MultimodalYieldConfig
    base_model_prefix = "yield_model"

    def __init__(self, config: MultimodalYieldConfig):
        super().__init__(config)

        tm, ts = _stat(config.text_mean, config.text_std, config.text_dim)
        im, is_ = _stat(config.image_mean, config.image_std, config.image_dim)
        self.register_buffer("text_mean", tm)
        self.register_buffer("text_std", ts)
        self.register_buffer("image_mean", im)
        self.register_buffer("image_std", is_)

        self.yield_model = MultimodalConcat_TemporalTF(
            text_dim=config.text_dim,
            image_dim=config.image_dim,
            d_model=config.d_model,
            nhead=config.nhead,
            num_layers=config.num_layers,
            dim_ff=config.dim_ff,
            dropout=config.dropout,
            input_dropout=config.input_dropout,
            max_weeks=config.K,
            pool=config.pool,
        )
        self.post_init()

    def forward(self, text, image, img_mask, img_age, cutoff=None, labels=None, return_dict=True):
        T = text.shape[1]
        t = T if cutoff is None else min(int(cutoff), T)

        txt = (text[:, :t].float() - self.text_mean) / self.text_std
        img = (image[:, :t].float() - self.image_mean) / self.image_std
        msk = img_mask[:, :t].float()
        age = img_age[:, :t].float() / self.config.K  # weeks -> ~[0, 1]

        logits = self.yield_model(text=txt, image=img, img_mask=msk, img_age=age, horizon_idx=t, causal=True)
        predictions = logits * self.config.y_std + self.config.y_mean

        loss = None
        if labels is not None:
            loss = nn.functional.mse_loss(logits, (labels - self.config.y_mean) / self.config.y_std)

        if not return_dict:
            return (loss, logits, predictions)
        return MultimodalYieldOutput(loss=loss, logits=logits, predictions=predictions)