Tabular Regression
Transformers
Safetensors
yield-text-image
feature-extraction
crop-yield-estimation
digital-agriculture
multimodal
multi-temporal
regression
custom_code
Instructions to use ICICLE-AI/multimodal-yield-estimation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ICICLE-AI/multimodal-yield-estimation with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("ICICLE-AI/multimodal-yield-estimation", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download modeling_multimodal.py from ICICLE-AI/multimodal-yield-estimation: direct link, hf CLI and curl.
- Browser
- Download file 3.09 kB
-
https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/modeling_multimodal.py
- Command line
-
hf download hf://ICICLE-AI/multimodal-yield-estimation/modeling_multimodal.py
-
curl -L -o modeling_multimodal.py https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/modeling_multimodal.py
3.09 kB
| from dataclasses import dataclass | |
| import torch | |
| from torch import nn | |
| from transformers import PreTrainedModel | |
| from transformers.modeling_outputs import ModelOutput | |
| from .configuration_multimodal import MultimodalYieldConfig | |
| from .multimodal_concat_tf import MultimodalConcat_TemporalTF | |
| class MultimodalYieldOutput(ModelOutput): | |
| loss: torch.Tensor | None = None | |
| logits: torch.Tensor | None = None | |
| predictions: torch.Tensor | None = None | |
| def _stat(mean, std, dim): | |
| if mean is None: | |
| return torch.zeros(dim), torch.ones(dim) | |
| return torch.tensor(mean, dtype=torch.float32), torch.tensor(std, dtype=torch.float32) | |
| class MultimodalYieldForRegression(PreTrainedModel): | |
| """Inputs (raw, un-normalized): | |
| text [B, T, 1024] weekly Qwen3 caption embeddings | |
| image [B, T, 768] DINOv3 feature of latest image at/before each week (any value where img_mask=0) | |
| img_mask [B, T] 1 if an image is available at/before that week | |
| img_age [B, T] weeks since that image was taken (0 = new image this week) | |
| cutoff int use only the first `cutoff` weeks after planting (defaults to T) | |
| """ | |
| config_class = MultimodalYieldConfig | |
| base_model_prefix = "yield_model" | |
| def __init__(self, config: MultimodalYieldConfig): | |
| super().__init__(config) | |
| tm, ts = _stat(config.text_mean, config.text_std, config.text_dim) | |
| im, is_ = _stat(config.image_mean, config.image_std, config.image_dim) | |
| self.register_buffer("text_mean", tm) | |
| self.register_buffer("text_std", ts) | |
| self.register_buffer("image_mean", im) | |
| self.register_buffer("image_std", is_) | |
| self.yield_model = MultimodalConcat_TemporalTF( | |
| text_dim=config.text_dim, | |
| image_dim=config.image_dim, | |
| d_model=config.d_model, | |
| nhead=config.nhead, | |
| num_layers=config.num_layers, | |
| dim_ff=config.dim_ff, | |
| dropout=config.dropout, | |
| input_dropout=config.input_dropout, | |
| max_weeks=config.K, | |
| pool=config.pool, | |
| ) | |
| self.post_init() | |
| def forward(self, text, image, img_mask, img_age, cutoff=None, labels=None, return_dict=True): | |
| T = text.shape[1] | |
| t = T if cutoff is None else min(int(cutoff), T) | |
| txt = (text[:, :t].float() - self.text_mean) / self.text_std | |
| img = (image[:, :t].float() - self.image_mean) / self.image_std | |
| msk = img_mask[:, :t].float() | |
| age = img_age[:, :t].float() / self.config.K # weeks -> ~[0, 1] | |
| logits = self.yield_model(text=txt, image=img, img_mask=msk, img_age=age, horizon_idx=t, causal=True) | |
| predictions = logits * self.config.y_std + self.config.y_mean | |
| loss = None | |
| if labels is not None: | |
| loss = nn.functional.mse_loss(logits, (labels - self.config.y_mean) / self.config.y_std) | |
| if not return_dict: | |
| return (loss, logits, predictions) | |
| return MultimodalYieldOutput(loss=loss, logits=logits, predictions=predictions) | |