"""Train the text + image (concat) multi-cutoff yield model as a HF model. Inputs per week: Qwen3 caption embedding (1024) + DINOv3 image feature (768) + has-image flag + image age. Split: one farm_field is held out (--test_field, default: last field alphabetically) and its plots are split 50/50 into val (early stopping) and test; all other fields are used for training. The best-val checkpoint is saved in Hugging Face format to --out_dir. """ import sys from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(PROJECT_ROOT)) import argparse import json import random import time import numpy as np import pandas as pd import torch from sklearn.metrics import r2_score from torch.utils.data import DataLoader from transformers import AutoModel from data.multimodal_dataset import MultimodalYieldDataset, compute_norm_stats, field_split, load_npz from hf.auto import register_multimodal_autoclass from hf.configuration_multimodal import MultimodalYieldConfig def parse_args(): p = argparse.ArgumentParser() p.add_argument("--data", required=True, help="embeddings .npz (see README)") p.add_argument("--test_field", default=None, help="farm_field held out for val/test (default: last field alphabetically); all others train") p.add_argument("--cutoffs", default="4,8,12,16,20,22") p.add_argument("--out_dir", default="checkpoints") p.add_argument("--epochs", type=int, default=300) p.add_argument("--patience", type=int, default=30) p.add_argument("--batch_size", type=int, default=16) p.add_argument("--lr", type=float, default=3e-4) p.add_argument("--weight_decay", type=float, default=1e-2) p.add_argument("--d_model", type=int, default=64) p.add_argument("--num_layers", type=int, default=2) p.add_argument("--pool", default="last", choices=["last", "mean"]) p.add_argument("--nhead", type=int, default=4) p.add_argument("--dim_ff", type=int, default=128) p.add_argument("--dropout", type=float, default=0.3) p.add_argument("--input_dropout", type=float, default=0.2) p.add_argument("--seed", type=int, default=1234) return p.parse_args() def set_seed(s): random.seed(s) np.random.seed(s) torch.manual_seed(s) torch.cuda.manual_seed_all(s) def to_dev(batch, device): return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in batch.items()} def train_one_epoch(model, loader, opt, device, cutoffs): model.train() tot, n = 0.0, 0 for b in loader: b = to_dev(b, device) losses = [ model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t, labels=b["yield"]).loss for t in cutoffs ] loss = torch.stack(losses).mean() opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) opt.step() tot += loss.item() * len(b["yield"]) n += len(b["yield"]) return tot / max(n, 1) @torch.no_grad() def evaluate(model, ds, device, cutoffs, batch_size=64): model.eval() loader = DataLoader(ds, batch_size=batch_size, shuffle=False) res, rows = {}, [] for t in cutoffs: yt, yp = [], [] for b in loader: b = to_dev(b, device) out = model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t) yt.append(b["yield"].cpu().numpy()) yp.append(out.predictions.cpu().numpy()) yt, yp = np.concatenate(yt), np.concatenate(yp) err = yp - yt res[int(t)] = { "rmse": float(np.sqrt(np.mean(err ** 2))), "mae": float(np.mean(np.abs(err))), "bias": float(np.mean(err)), "r2": float(r2_score(yt, yp)) if len(yt) > 1 else float("nan"), } for i in range(len(yt)): rows.append({ "cutoff": int(t), "farm_field": ds.meta["farm_field"][i], "plot": int(ds.meta["plot"][i]), "row_key": ds.meta["row_key"][i], "y_true": float(yt[i]), "y_pred": float(yp[i]), "error": float(err[i]), }) return res, rows def print_table(title, res): print(f"\n{title}\n{'Cutoff':>8}{'RMSE':>10}{'MAE':>10}{'Bias':>10}{'R2':>10}") for t, m in res.items(): print(f"{t:>8}{m['rmse']:>10.3f}{m['mae']:>10.3f}{m['bias']:>10.3f}{m['r2']:>10.3f}") def save_json(path, obj): with open(path, "w") as f: json.dump(obj, f, indent=2, default=str) def main(): a = parse_args() set_seed(a.seed) register_multimodal_autoclass() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") out = Path(a.out_dir) out.mkdir(parents=True, exist_ok=True) save_json(out / "train_args.json", vars(a)) cutoffs = [int(c) for c in a.cutoffs.split(",")] data = load_npz(a.data) train_idx, val_idx, test_idx, train_fields, test_field = field_split(data, a.test_field, seed=a.seed) print(f"device={device} train_fields={train_fields} test_field={test_field} " f"train={len(train_idx)} val={len(val_idx)} test={len(test_idx)}") stats = compute_norm_stats(data, train_idx) cfg = MultimodalYieldConfig( K=int(data["text"].shape[1]), text_dim=int(data["text"].shape[2]), image_dim=int(data["image"].shape[2]), train_cutoffs=cutoffs, eval_cutoffs=cutoffs, d_model=a.d_model, nhead=a.nhead, num_layers=a.num_layers, dim_ff=a.dim_ff, dropout=a.dropout, input_dropout=a.input_dropout, pool=a.pool, **stats, ) model = AutoModel.from_config(cfg).to(device) print(f"params: {sum(p.numel() for p in model.parameters()):,}") train_ds = MultimodalYieldDataset(data, train_idx) loader = DataLoader(train_ds, batch_size=a.batch_size, shuffle=True, drop_last=len(train_ds) > a.batch_size) opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=a.weight_decay) val_ds = MultimodalYieldDataset(data, val_idx) test_ds = MultimodalYieldDataset(data, test_idx) best, bad, best_ep = float("inf"), 0, 0 t0 = time.time() for ep in range(1, a.epochs + 1): loss = train_one_epoch(model, loader, opt, device, cutoffs) val_res, _ = evaluate(model, val_ds, device, cutoffs) val_rmse = float(np.mean([m["rmse"] for m in val_res.values()])) if val_rmse < best: best, bad, best_ep = val_rmse, 0, ep model.save_pretrained(out) save_json(out / "metrics.json", {"epoch": ep, "val_rmse": val_rmse, "val_by_cutoff": val_res}) else: bad += 1 if ep % 10 == 0 or bad == 0: print(f"epoch {ep:03d} train_loss={loss:.4f} val_rmse={val_rmse:.3f} best={best:.3f}@{best_ep}") if bad >= a.patience: print(f"early stop at epoch {ep}") break print(f"training time {time.time() - t0:.1f}s, best epoch {best_ep}") best_model = AutoModel.from_pretrained(out).to(device) val_res, _ = evaluate(best_model, val_ds, device, cutoffs) test_res, rows = evaluate(best_model, test_ds, device, cutoffs) print_table(f"Val metrics ({test_field})", val_res) print_table(f"Test metrics ({test_field})", test_res) # Reference (log only): RMSE of predicting the training-set mean yield yt = data["yield"][test_idx] print(f"\nTrain-mean baseline test RMSE: {np.sqrt(np.mean((yt - stats['y_mean']) ** 2)):.3f}") save_json(out / "test_metrics.json", { "train_fields": train_fields, "test_field": test_field, "best_epoch": best_ep, "n_train": len(train_idx), "n_val": len(val_idx), "n_test": len(test_idx), "test_by_cutoff": test_res, "val_by_cutoff": val_res, }) pd.DataFrame(rows).to_csv(out / "test_predictions.csv", index=False) print(f"saved to {out}") if __name__ == "__main__": main()