Tabular Regression
Transformers
Safetensors
yield-text-image
feature-extraction
crop-yield-estimation
digital-agriculture
multimodal
multi-temporal
regression
custom_code
Instructions to use ICICLE-AI/multimodal-yield-estimation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ICICLE-AI/multimodal-yield-estimation with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("ICICLE-AI/multimodal-yield-estimation", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download scripts/train_multimodal_hf.py from ICICLE-AI/multimodal-yield-estimation: direct link, hf CLI and curl.
- Browser
- Download file 8.02 kB
-
https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/scripts/train_multimodal_hf.py
- Command line
-
hf download hf://ICICLE-AI/multimodal-yield-estimation/scripts/train_multimodal_hf.py
-
curl -L -o train_multimodal_hf.py https://huggingface.co/ICICLE-AI/multimodal-yield-estimation/resolve/main/scripts/train_multimodal_hf.py
8.02 kB
| """Train the text + image (concat) multi-cutoff yield model as a HF model. | |
| Inputs per week: Qwen3 caption embedding (1024) + DINOv3 image feature (768) + has-image flag + image age. | |
| Split: one farm_field is held out (--test_field, default: last field alphabetically) and its | |
| plots are split 50/50 into val (early stopping) and test; all other fields are used for training. | |
| The best-val checkpoint is saved in Hugging Face format to --out_dir. | |
| """ | |
| import sys | |
| from pathlib import Path | |
| PROJECT_ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(PROJECT_ROOT)) | |
| import argparse | |
| import json | |
| import random | |
| import time | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from sklearn.metrics import r2_score | |
| from torch.utils.data import DataLoader | |
| from transformers import AutoModel | |
| from data.multimodal_dataset import MultimodalYieldDataset, compute_norm_stats, field_split, load_npz | |
| from hf.auto import register_multimodal_autoclass | |
| from hf.configuration_multimodal import MultimodalYieldConfig | |
| def parse_args(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--data", required=True, help="embeddings .npz (see README)") | |
| p.add_argument("--test_field", default=None, | |
| help="farm_field held out for val/test (default: last field alphabetically); all others train") | |
| p.add_argument("--cutoffs", default="4,8,12,16,20,22") | |
| p.add_argument("--out_dir", default="checkpoints") | |
| p.add_argument("--epochs", type=int, default=300) | |
| p.add_argument("--patience", type=int, default=30) | |
| p.add_argument("--batch_size", type=int, default=16) | |
| p.add_argument("--lr", type=float, default=3e-4) | |
| p.add_argument("--weight_decay", type=float, default=1e-2) | |
| p.add_argument("--d_model", type=int, default=64) | |
| p.add_argument("--num_layers", type=int, default=2) | |
| p.add_argument("--pool", default="last", choices=["last", "mean"]) | |
| p.add_argument("--nhead", type=int, default=4) | |
| p.add_argument("--dim_ff", type=int, default=128) | |
| p.add_argument("--dropout", type=float, default=0.3) | |
| p.add_argument("--input_dropout", type=float, default=0.2) | |
| p.add_argument("--seed", type=int, default=1234) | |
| return p.parse_args() | |
| def set_seed(s): | |
| random.seed(s) | |
| np.random.seed(s) | |
| torch.manual_seed(s) | |
| torch.cuda.manual_seed_all(s) | |
| def to_dev(batch, device): | |
| return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in batch.items()} | |
| def train_one_epoch(model, loader, opt, device, cutoffs): | |
| model.train() | |
| tot, n = 0.0, 0 | |
| for b in loader: | |
| b = to_dev(b, device) | |
| losses = [ | |
| model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t, labels=b["yield"]).loss | |
| for t in cutoffs | |
| ] | |
| loss = torch.stack(losses).mean() | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) | |
| opt.step() | |
| tot += loss.item() * len(b["yield"]) | |
| n += len(b["yield"]) | |
| return tot / max(n, 1) | |
| def evaluate(model, ds, device, cutoffs, batch_size=64): | |
| model.eval() | |
| loader = DataLoader(ds, batch_size=batch_size, shuffle=False) | |
| res, rows = {}, [] | |
| for t in cutoffs: | |
| yt, yp = [], [] | |
| for b in loader: | |
| b = to_dev(b, device) | |
| out = model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t) | |
| yt.append(b["yield"].cpu().numpy()) | |
| yp.append(out.predictions.cpu().numpy()) | |
| yt, yp = np.concatenate(yt), np.concatenate(yp) | |
| err = yp - yt | |
| res[int(t)] = { | |
| "rmse": float(np.sqrt(np.mean(err ** 2))), | |
| "mae": float(np.mean(np.abs(err))), | |
| "bias": float(np.mean(err)), | |
| "r2": float(r2_score(yt, yp)) if len(yt) > 1 else float("nan"), | |
| } | |
| for i in range(len(yt)): | |
| rows.append({ | |
| "cutoff": int(t), | |
| "farm_field": ds.meta["farm_field"][i], | |
| "plot": int(ds.meta["plot"][i]), | |
| "row_key": ds.meta["row_key"][i], | |
| "y_true": float(yt[i]), | |
| "y_pred": float(yp[i]), | |
| "error": float(err[i]), | |
| }) | |
| return res, rows | |
| def print_table(title, res): | |
| print(f"\n{title}\n{'Cutoff':>8}{'RMSE':>10}{'MAE':>10}{'Bias':>10}{'R2':>10}") | |
| for t, m in res.items(): | |
| print(f"{t:>8}{m['rmse']:>10.3f}{m['mae']:>10.3f}{m['bias']:>10.3f}{m['r2']:>10.3f}") | |
| def save_json(path, obj): | |
| with open(path, "w") as f: | |
| json.dump(obj, f, indent=2, default=str) | |
| def main(): | |
| a = parse_args() | |
| set_seed(a.seed) | |
| register_multimodal_autoclass() | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| out = Path(a.out_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| save_json(out / "train_args.json", vars(a)) | |
| cutoffs = [int(c) for c in a.cutoffs.split(",")] | |
| data = load_npz(a.data) | |
| train_idx, val_idx, test_idx, train_fields, test_field = field_split(data, a.test_field, seed=a.seed) | |
| print(f"device={device} train_fields={train_fields} test_field={test_field} " | |
| f"train={len(train_idx)} val={len(val_idx)} test={len(test_idx)}") | |
| stats = compute_norm_stats(data, train_idx) | |
| cfg = MultimodalYieldConfig( | |
| K=int(data["text"].shape[1]), | |
| text_dim=int(data["text"].shape[2]), | |
| image_dim=int(data["image"].shape[2]), | |
| train_cutoffs=cutoffs, | |
| eval_cutoffs=cutoffs, | |
| d_model=a.d_model, nhead=a.nhead, num_layers=a.num_layers, dim_ff=a.dim_ff, | |
| dropout=a.dropout, input_dropout=a.input_dropout, pool=a.pool, | |
| **stats, | |
| ) | |
| model = AutoModel.from_config(cfg).to(device) | |
| print(f"params: {sum(p.numel() for p in model.parameters()):,}") | |
| train_ds = MultimodalYieldDataset(data, train_idx) | |
| loader = DataLoader(train_ds, batch_size=a.batch_size, shuffle=True, drop_last=len(train_ds) > a.batch_size) | |
| opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=a.weight_decay) | |
| val_ds = MultimodalYieldDataset(data, val_idx) | |
| test_ds = MultimodalYieldDataset(data, test_idx) | |
| best, bad, best_ep = float("inf"), 0, 0 | |
| t0 = time.time() | |
| for ep in range(1, a.epochs + 1): | |
| loss = train_one_epoch(model, loader, opt, device, cutoffs) | |
| val_res, _ = evaluate(model, val_ds, device, cutoffs) | |
| val_rmse = float(np.mean([m["rmse"] for m in val_res.values()])) | |
| if val_rmse < best: | |
| best, bad, best_ep = val_rmse, 0, ep | |
| model.save_pretrained(out) | |
| save_json(out / "metrics.json", {"epoch": ep, "val_rmse": val_rmse, "val_by_cutoff": val_res}) | |
| else: | |
| bad += 1 | |
| if ep % 10 == 0 or bad == 0: | |
| print(f"epoch {ep:03d} train_loss={loss:.4f} val_rmse={val_rmse:.3f} best={best:.3f}@{best_ep}") | |
| if bad >= a.patience: | |
| print(f"early stop at epoch {ep}") | |
| break | |
| print(f"training time {time.time() - t0:.1f}s, best epoch {best_ep}") | |
| best_model = AutoModel.from_pretrained(out).to(device) | |
| val_res, _ = evaluate(best_model, val_ds, device, cutoffs) | |
| test_res, rows = evaluate(best_model, test_ds, device, cutoffs) | |
| print_table(f"Val metrics ({test_field})", val_res) | |
| print_table(f"Test metrics ({test_field})", test_res) | |
| # Reference (log only): RMSE of predicting the training-set mean yield | |
| yt = data["yield"][test_idx] | |
| print(f"\nTrain-mean baseline test RMSE: {np.sqrt(np.mean((yt - stats['y_mean']) ** 2)):.3f}") | |
| save_json(out / "test_metrics.json", { | |
| "train_fields": train_fields, "test_field": test_field, "best_epoch": best_ep, | |
| "n_train": len(train_idx), "n_val": len(val_idx), "n_test": len(test_idx), | |
| "test_by_cutoff": test_res, "val_by_cutoff": val_res, | |
| }) | |
| pd.DataFrame(rows).to_csv(out / "test_predictions.csv", index=False) | |
| print(f"saved to {out}") | |
| if __name__ == "__main__": | |
| main() | |