"""Raw inputs -> model inputs: Qwen3 caption embeddings + DINOv3 image features. Matches how the training features were made: - text : Qwen/Qwen3-Embedding-0.6B via SentenceTransformer, L2-normalized, stored as fp16 - image: facebook/dinov3-vitb16-pretrain-lvd1689m, AutoImageProcessor, pooler_output (CLS fallback) Sample format (one dict per plot): { "sample_id": "plot101", # optional "planting_date": "2024-05-16", "week": 12, # cutoff: number of weeks after planting to use (1..22) "texts": ["", ...], # weekly captions, at least `week` of them "images": [{"path": "...tif", "date": "2024-05-30"}, ...] } """ from __future__ import annotations import numpy as np import torch from PIL import Image from data.multimodal_dataset import N_WEEKS, images_to_weeks TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B" IMAGE_MODEL = "facebook/dinov3-vitb16-pretrain-lvd1689m" class TextEmbedder: def __init__(self, model_name: str = TEXT_MODEL, device=None): from sentence_transformers import SentenceTransformer self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu")) self.model = SentenceTransformer(model_name, trust_remote_code=True, device=str(self.device)) if self.device.type == "cuda": self.model = self.model.half() def encode(self, texts: list[str], batch_size: int = 128) -> np.ndarray: unique, inverse = np.unique(np.asarray(texts, dtype=object), return_inverse=True) emb = self.model.encode( unique.tolist(), batch_size=batch_size, normalize_embeddings=True, convert_to_numpy=True, show_progress_bar=False, ) return emb[inverse].astype(np.float16).astype(np.float32) class ImageEmbedder: def __init__(self, model_name: str = IMAGE_MODEL, device=None): from transformers import AutoImageProcessor, AutoModel self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu")) if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device) < (7, 5): torch.backends.cudnn.enabled = False self.processor = AutoImageProcessor.from_pretrained(model_name) self.model = AutoModel.from_pretrained(model_name).to(self.device).eval() @torch.inference_mode() def encode(self, paths: list[str], batch_size: int = 32) -> np.ndarray: feats = [] for s in range(0, len(paths), batch_size): images = [Image.open(p).convert("RGB") for p in paths[s:s + batch_size]] inputs = self.processor(images=images, return_tensors="pt") out = self.model(**{k: v.to(self.device) for k, v in inputs.items()}) f = out.pooler_output if getattr(out, "pooler_output", None) is not None else out.last_hidden_state[:, 0] feats.append(f.float().cpu()) if not feats: return np.zeros((0, self.model.config.hidden_size), dtype=np.float32) # no images: all weeks masked return torch.cat(feats).numpy() def embed_samples(samples: list[dict], text_embedder: TextEmbedder, image_embedder: ImageEmbedder, week: int | None = None): texts = [t for s in samples for t in s["texts"]] paths = [im["path"] for s in samples for im in s.get("images", [])] text_emb = text_embedder.encode(texts) image_feats = image_embedder.encode(paths) out = {"text": [], "image": [], "img_mask": [], "img_age": [], "week": []} ti = ii = 0 for s in samples: nt, ni = len(s["texts"]), len(s.get("images", [])) text, image, img_mask, img_age, w = build_inputs(s, text_emb[ti:ti + nt], image_feats[ii:ii + ni], week) ti, ii = ti + nt, ii + ni out["text"].append(text) out["image"].append(image) out["img_mask"].append(img_mask) out["img_age"].append(img_age) out["week"].append(w) return {k: np.stack(v) for k, v in out.items()} def build_inputs(sample: dict, text_emb: np.ndarray, image_feats: np.ndarray, week: int | None = None): """Embedded sample -> (text [22,Dt], image [22,Di], img_mask [22], img_age [22], week).""" week = int(week or sample["week"]) if not 1 <= week <= N_WEEKS: raise ValueError(f"week must be in 1..{N_WEEKS}, got {week}") if len(text_emb) < week: raise ValueError(f"need captions for weeks 0..{week - 1}, got {len(text_emb)}") text = np.zeros((N_WEEKS, text_emb.shape[1]), dtype=np.float32) n = min(len(text_emb), N_WEEKS) text[:n] = text_emb[:n] dates = [im["date"] for im in sample.get("images", [])] image, img_mask, img_age, _ = images_to_weeks(dates, image_feats, sample["planting_date"]) return text, image, img_mask, img_age, week