File size: 4,814 Bytes
4191737
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""Raw inputs -> model inputs: Qwen3 caption embeddings + DINOv3 image features.

Matches how the training features were made:
- text : Qwen/Qwen3-Embedding-0.6B via SentenceTransformer, L2-normalized, stored as fp16
- image: facebook/dinov3-vitb16-pretrain-lvd1689m, AutoImageProcessor, pooler_output (CLS fallback)

Sample format (one dict per plot):
    {
      "sample_id": "plot101",                 # optional
      "planting_date": "2024-05-16",
      "week": 12,                             # cutoff: number of weeks after planting to use (1..22)
      "texts": ["<week 0 caption>", ...],     # weekly captions, at least `week` of them
      "images": [{"path": "...tif", "date": "2024-05-30"}, ...]
    }
"""
from __future__ import annotations

import numpy as np
import torch
from PIL import Image

from data.multimodal_dataset import N_WEEKS, images_to_weeks

TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B"
IMAGE_MODEL = "facebook/dinov3-vitb16-pretrain-lvd1689m"


class TextEmbedder:
    def __init__(self, model_name: str = TEXT_MODEL, device=None):
        from sentence_transformers import SentenceTransformer

        self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
        self.model = SentenceTransformer(model_name, trust_remote_code=True, device=str(self.device))
        if self.device.type == "cuda":
            self.model = self.model.half()

    def encode(self, texts: list[str], batch_size: int = 128) -> np.ndarray:
        unique, inverse = np.unique(np.asarray(texts, dtype=object), return_inverse=True)
        emb = self.model.encode(
            unique.tolist(), batch_size=batch_size, normalize_embeddings=True,
            convert_to_numpy=True, show_progress_bar=False,
        )
        return emb[inverse].astype(np.float16).astype(np.float32)


class ImageEmbedder:
    def __init__(self, model_name: str = IMAGE_MODEL, device=None):
        from transformers import AutoImageProcessor, AutoModel

        self.device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
        if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device) < (7, 5):
            torch.backends.cudnn.enabled = False
        self.processor = AutoImageProcessor.from_pretrained(model_name)
        self.model = AutoModel.from_pretrained(model_name).to(self.device).eval()

    @torch.inference_mode()
    def encode(self, paths: list[str], batch_size: int = 32) -> np.ndarray:
        feats = []
        for s in range(0, len(paths), batch_size):
            images = [Image.open(p).convert("RGB") for p in paths[s:s + batch_size]]
            inputs = self.processor(images=images, return_tensors="pt")
            out = self.model(**{k: v.to(self.device) for k, v in inputs.items()})
            f = out.pooler_output if getattr(out, "pooler_output", None) is not None else out.last_hidden_state[:, 0]
            feats.append(f.float().cpu())
        if not feats:
            return np.zeros((0, self.model.config.hidden_size), dtype=np.float32)  # no images: all weeks masked
        return torch.cat(feats).numpy()


def embed_samples(samples: list[dict], text_embedder: TextEmbedder, image_embedder: ImageEmbedder, week: int | None = None):
    
    texts = [t for s in samples for t in s["texts"]]
    paths = [im["path"] for s in samples for im in s.get("images", [])]
    text_emb = text_embedder.encode(texts)
    image_feats = image_embedder.encode(paths)

    out = {"text": [], "image": [], "img_mask": [], "img_age": [], "week": []}
    ti = ii = 0
    for s in samples:
        nt, ni = len(s["texts"]), len(s.get("images", []))
        text, image, img_mask, img_age, w = build_inputs(s, text_emb[ti:ti + nt], image_feats[ii:ii + ni], week)
        ti, ii = ti + nt, ii + ni
        out["text"].append(text)
        out["image"].append(image)
        out["img_mask"].append(img_mask)
        out["img_age"].append(img_age)
        out["week"].append(w)
    return {k: np.stack(v) for k, v in out.items()}


def build_inputs(sample: dict, text_emb: np.ndarray, image_feats: np.ndarray, week: int | None = None):
    """Embedded sample -> (text [22,Dt], image [22,Di], img_mask [22], img_age [22], week)."""
    week = int(week or sample["week"])
    if not 1 <= week <= N_WEEKS:
        raise ValueError(f"week must be in 1..{N_WEEKS}, got {week}")
    if len(text_emb) < week:
        raise ValueError(f"need captions for weeks 0..{week - 1}, got {len(text_emb)}")

    text = np.zeros((N_WEEKS, text_emb.shape[1]), dtype=np.float32)
    n = min(len(text_emb), N_WEEKS)
    text[:n] = text_emb[:n]

    dates = [im["date"] for im in sample.get("images", [])]
    image, img_mask, img_age, _ = images_to_weeks(dates, image_feats, sample["planting_date"])
    return text, image, img_mask, img_age, week