Sarikaa-Sridhar's picture
multi-modal yield estimation code base
4191737 verified
Raw History Blame Contribute Delete
4.81 kB
"""Raw inputs -> model inputs: Qwen3 caption embeddings + DINOv3 image features.
Matches how the training features were made:
- text : Qwen/Qwen3-Embedding-0.6B via SentenceTransformer, L2-normalized, stored as fp16
- image: facebook/dinov3-vitb16-pretrain-lvd1689m, AutoImageProcessor, pooler_output (CLS fallback)
Sample format (one dict per plot):
{
"sample_id": "plot101", # optional
"planting_date": "2024-05-16",
"week": 12, # cutoff: number of weeks after planting to use (1..22)
"texts": ["<week 0 caption>", ...], # weekly captions, at least `week` of them
"images": [{"path": "...tif", "date": "2024-05-30"}, ...]
}
"""
from __future__ import annotations
import numpy as np
import torch
from PIL import Image
from data.multimodal_dataset import N_WEEKS, images_to_weeks
TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B"
IMAGE_MODEL = "facebook/dinov3-vitb16-pretrain-lvd1689m"
class TextEmbedder:
def __init__(self, model_name: str = TEXT_MODEL, device=None):
from sentence_transformers import SentenceTransformer
self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
self.model = SentenceTransformer(model_name, trust_remote_code=True, device=str(self.device))
if self.device.type == "cuda":
self.model = self.model.half()
def encode(self, texts: list[str], batch_size: int = 128) -> np.ndarray:
unique, inverse = np.unique(np.asarray(texts, dtype=object), return_inverse=True)
emb = self.model.encode(
unique.tolist(), batch_size=batch_size, normalize_embeddings=True,
convert_to_numpy=True, show_progress_bar=False,
)
return emb[inverse].astype(np.float16).astype(np.float32)
class ImageEmbedder:
def __init__(self, model_name: str = IMAGE_MODEL, device=None):
from transformers import AutoImageProcessor, AutoModel
self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device) < (7, 5):
torch.backends.cudnn.enabled = False
self.processor = AutoImageProcessor.from_pretrained(model_name)
self.model = AutoModel.from_pretrained(model_name).to(self.device).eval()
@torch.inference_mode()
def encode(self, paths: list[str], batch_size: int = 32) -> np.ndarray:
feats = []
for s in range(0, len(paths), batch_size):
images = [Image.open(p).convert("RGB") for p in paths[s:s + batch_size]]
inputs = self.processor(images=images, return_tensors="pt")
out = self.model(**{k: v.to(self.device) for k, v in inputs.items()})
f = out.pooler_output if getattr(out, "pooler_output", None) is not None else out.last_hidden_state[:, 0]
feats.append(f.float().cpu())
if not feats:
return np.zeros((0, self.model.config.hidden_size), dtype=np.float32) # no images: all weeks masked
return torch.cat(feats).numpy()
def embed_samples(samples: list[dict], text_embedder: TextEmbedder, image_embedder: ImageEmbedder, week: int | None = None):
texts = [t for s in samples for t in s["texts"]]
paths = [im["path"] for s in samples for im in s.get("images", [])]
text_emb = text_embedder.encode(texts)
image_feats = image_embedder.encode(paths)
out = {"text": [], "image": [], "img_mask": [], "img_age": [], "week": []}
ti = ii = 0
for s in samples:
nt, ni = len(s["texts"]), len(s.get("images", []))
text, image, img_mask, img_age, w = build_inputs(s, text_emb[ti:ti + nt], image_feats[ii:ii + ni], week)
ti, ii = ti + nt, ii + ni
out["text"].append(text)
out["image"].append(image)
out["img_mask"].append(img_mask)
out["img_age"].append(img_age)
out["week"].append(w)
return {k: np.stack(v) for k, v in out.items()}
def build_inputs(sample: dict, text_emb: np.ndarray, image_feats: np.ndarray, week: int | None = None):
"""Embedded sample -> (text [22,Dt], image [22,Di], img_mask [22], img_age [22], week)."""
week = int(week or sample["week"])
if not 1 <= week <= N_WEEKS:
raise ValueError(f"week must be in 1..{N_WEEKS}, got {week}")
if len(text_emb) < week:
raise ValueError(f"need captions for weeks 0..{week - 1}, got {len(text_emb)}")
text = np.zeros((N_WEEKS, text_emb.shape[1]), dtype=np.float32)
n = min(len(text_emb), N_WEEKS)
text[:n] = text_emb[:n]
dates = [im["date"] for im in sample.get("images", [])]
image, img_mask, img_age, _ = images_to_weeks(dates, image_feats, sample["planting_date"])
return text, image, img_mask, img_age, week