Tabular Regression
Transformers
Safetensors
yield-text-image
feature-extraction
crop-yield-estimation
digital-agriculture
multimodal
multi-temporal
regression
custom_code
Instructions to use ICICLE-AI/multimodal-yield-estimation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ICICLE-AI/multimodal-yield-estimation with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("ICICLE-AI/multimodal-yield-estimation", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download multimodal_concat_tf.py from ICICLE-AI/multimodal-yield-estimation: direct link, hf CLI and curl.
- Browser
- Download file 3.25 kB
-
https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/multimodal_concat_tf.py
- Command line
-
hf download hf://ICICLE-AI/multimodal-yield-estimation/multimodal_concat_tf.py
-
curl -L -o multimodal_concat_tf.py https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/multimodal_concat_tf.py
3.25 kB
| import torch | |
| import torch.nn as nn | |
| class MultimodalConcat_TemporalTF(nn.Module): | |
| """Weekly tokens = concat(text_emb_w, image_emb_w, has_image_w, image_age_w) -> causal temporal transformer. | |
| Horizon embedding, week signal + learned positions, causal TransformerEncoder, | |
| last-token pooling and a small regression head. | |
| """ | |
| def __init__( | |
| self, | |
| text_dim: int = 1024, | |
| image_dim: int = 768, | |
| d_model: int = 64, | |
| nhead: int = 4, | |
| num_layers: int = 2, | |
| dim_ff: int = 128, | |
| dropout: float = 0.3, | |
| input_dropout: float = 0.2, | |
| max_weeks: int = 22, | |
| pool: str = "last", | |
| horizon_emb_dim: int = 16, | |
| ): | |
| super().__init__() | |
| assert pool in ["last", "mean"] | |
| self.pool = pool | |
| self.max_weeks = max_weeks | |
| in_dim = text_dim + image_dim + 2 | |
| self.input_proj = nn.Sequential( | |
| nn.Dropout(input_dropout), | |
| nn.Linear(in_dim, d_model), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(d_model, d_model), | |
| ) | |
| self.in_ln = nn.LayerNorm(d_model) | |
| self.horizon_emb = nn.Embedding(max_weeks + 1, horizon_emb_dim) | |
| self.horizon_proj = nn.Linear(horizon_emb_dim, d_model) | |
| self.week_proj = nn.Linear(1, d_model) | |
| self.pos_emb = nn.Parameter(torch.zeros(1, max_weeks + 1, d_model)) | |
| nn.init.trunc_normal_(self.pos_emb, std=0.02) | |
| enc_layer = nn.TransformerEncoderLayer( | |
| d_model=d_model, nhead=nhead, dim_feedforward=dim_ff, | |
| dropout=dropout, activation="gelu", batch_first=True, norm_first=True, | |
| ) | |
| self.temporal_tf = nn.TransformerEncoder(enc_layer, num_layers=num_layers, enable_nested_tensor=False) | |
| self.head = nn.Sequential( | |
| nn.LayerNorm(d_model + horizon_emb_dim), | |
| nn.Linear(d_model + horizon_emb_dim, d_model), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(d_model, 1), | |
| ) | |
| def forward(self, text, image, img_mask, img_age, horizon_idx=None, causal=True): | |
| m = img_mask.unsqueeze(-1).to(image.dtype) | |
| age = img_age.unsqueeze(-1).to(image.dtype) | |
| x = torch.cat([text, image * m, m, age * m], dim=-1) | |
| B, t, _ = x.shape | |
| device = x.device | |
| if horizon_idx is None: | |
| horizon_idx = t | |
| if not torch.is_tensor(horizon_idx): | |
| horizon_idx = torch.tensor(horizon_idx, device=device) | |
| h_idx = horizon_idx.long().expand(B).clamp(min=1, max=self.max_weeks) | |
| h_emb = self.horizon_emb(h_idx) | |
| h_tok = self.horizon_proj(h_emb) | |
| tok = self.in_ln(self.input_proj(x)) + h_tok[:, None, :] | |
| weeks = torch.arange(1, t + 1, device=device).float() / self.max_weeks | |
| tok = tok + self.week_proj(weeks[None, :, None].expand(B, -1, -1)) + self.pos_emb[:, :t, :] | |
| attn_mask = None | |
| if causal: | |
| attn_mask = torch.triu(torch.ones(t, t, device=device, dtype=torch.bool), diagonal=1) | |
| h = self.temporal_tf(tok, mask=attn_mask) | |
| pooled = h[:, -1, :] if self.pool == "last" else h.mean(dim=1) | |
| return self.head(torch.cat([pooled, h_emb], dim=-1)).squeeze(-1) | |