Tabular Regression
Transformers
Safetensors
yield-text-image
feature-extraction
crop-yield-estimation
digital-agriculture
multimodal
multi-temporal
regression
custom_code
Instructions to use ICICLE-AI/multimodal-yield-estimation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ICICLE-AI/multimodal-yield-estimation with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("ICICLE-AI/multimodal-yield-estimation", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download data/embeddings.py from ICICLE-AI/multimodal-yield-estimation: direct link, hf CLI and curl.
- Browser
- Download file 4.81 kB
-
https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/data/embeddings.py
- Command line
-
hf download hf://ICICLE-AI/multimodal-yield-estimation/data/embeddings.py
-
curl -L -o embeddings.py https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/data/embeddings.py
4.81 kB
| """Raw inputs -> model inputs: Qwen3 caption embeddings + DINOv3 image features. | |
| Matches how the training features were made: | |
| - text : Qwen/Qwen3-Embedding-0.6B via SentenceTransformer, L2-normalized, stored as fp16 | |
| - image: facebook/dinov3-vitb16-pretrain-lvd1689m, AutoImageProcessor, pooler_output (CLS fallback) | |
| Sample format (one dict per plot): | |
| { | |
| "sample_id": "plot101", # optional | |
| "planting_date": "2024-05-16", | |
| "week": 12, # cutoff: number of weeks after planting to use (1..22) | |
| "texts": ["<week 0 caption>", ...], # weekly captions, at least `week` of them | |
| "images": [{"path": "...tif", "date": "2024-05-30"}, ...] | |
| } | |
| """ | |
| from __future__ import annotations | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from data.multimodal_dataset import N_WEEKS, images_to_weeks | |
| TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B" | |
| IMAGE_MODEL = "facebook/dinov3-vitb16-pretrain-lvd1689m" | |
| class TextEmbedder: | |
| def __init__(self, model_name: str = TEXT_MODEL, device=None): | |
| from sentence_transformers import SentenceTransformer | |
| self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu")) | |
| self.model = SentenceTransformer(model_name, trust_remote_code=True, device=str(self.device)) | |
| if self.device.type == "cuda": | |
| self.model = self.model.half() | |
| def encode(self, texts: list[str], batch_size: int = 128) -> np.ndarray: | |
| unique, inverse = np.unique(np.asarray(texts, dtype=object), return_inverse=True) | |
| emb = self.model.encode( | |
| unique.tolist(), batch_size=batch_size, normalize_embeddings=True, | |
| convert_to_numpy=True, show_progress_bar=False, | |
| ) | |
| return emb[inverse].astype(np.float16).astype(np.float32) | |
| class ImageEmbedder: | |
| def __init__(self, model_name: str = IMAGE_MODEL, device=None): | |
| from transformers import AutoImageProcessor, AutoModel | |
| self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu")) | |
| if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device) < (7, 5): | |
| torch.backends.cudnn.enabled = False | |
| self.processor = AutoImageProcessor.from_pretrained(model_name) | |
| self.model = AutoModel.from_pretrained(model_name).to(self.device).eval() | |
| def encode(self, paths: list[str], batch_size: int = 32) -> np.ndarray: | |
| feats = [] | |
| for s in range(0, len(paths), batch_size): | |
| images = [Image.open(p).convert("RGB") for p in paths[s:s + batch_size]] | |
| inputs = self.processor(images=images, return_tensors="pt") | |
| out = self.model(**{k: v.to(self.device) for k, v in inputs.items()}) | |
| f = out.pooler_output if getattr(out, "pooler_output", None) is not None else out.last_hidden_state[:, 0] | |
| feats.append(f.float().cpu()) | |
| if not feats: | |
| return np.zeros((0, self.model.config.hidden_size), dtype=np.float32) # no images: all weeks masked | |
| return torch.cat(feats).numpy() | |
| def embed_samples(samples: list[dict], text_embedder: TextEmbedder, image_embedder: ImageEmbedder, week: int | None = None): | |
| texts = [t for s in samples for t in s["texts"]] | |
| paths = [im["path"] for s in samples for im in s.get("images", [])] | |
| text_emb = text_embedder.encode(texts) | |
| image_feats = image_embedder.encode(paths) | |
| out = {"text": [], "image": [], "img_mask": [], "img_age": [], "week": []} | |
| ti = ii = 0 | |
| for s in samples: | |
| nt, ni = len(s["texts"]), len(s.get("images", [])) | |
| text, image, img_mask, img_age, w = build_inputs(s, text_emb[ti:ti + nt], image_feats[ii:ii + ni], week) | |
| ti, ii = ti + nt, ii + ni | |
| out["text"].append(text) | |
| out["image"].append(image) | |
| out["img_mask"].append(img_mask) | |
| out["img_age"].append(img_age) | |
| out["week"].append(w) | |
| return {k: np.stack(v) for k, v in out.items()} | |
| def build_inputs(sample: dict, text_emb: np.ndarray, image_feats: np.ndarray, week: int | None = None): | |
| """Embedded sample -> (text [22,Dt], image [22,Di], img_mask [22], img_age [22], week).""" | |
| week = int(week or sample["week"]) | |
| if not 1 <= week <= N_WEEKS: | |
| raise ValueError(f"week must be in 1..{N_WEEKS}, got {week}") | |
| if len(text_emb) < week: | |
| raise ValueError(f"need captions for weeks 0..{week - 1}, got {len(text_emb)}") | |
| text = np.zeros((N_WEEKS, text_emb.shape[1]), dtype=np.float32) | |
| n = min(len(text_emb), N_WEEKS) | |
| text[:n] = text_emb[:n] | |
| dates = [im["date"] for im in sample.get("images", [])] | |
| image, img_mask, img_age, _ = images_to_weeks(dates, image_feats, sample["planting_date"]) | |
| return text, image, img_mask, img_age, week | |