multimodal-yield-estimation / scripts /train_multimodal_hf.py
Sarikaa-Sridhar's picture
multi-modal yield estimation code base
4191737 verified
Raw History Blame Contribute Delete
8.02 kB
"""Train the text + image (concat) multi-cutoff yield model as a HF model.
Inputs per week: Qwen3 caption embedding (1024) + DINOv3 image feature (768) + has-image flag + image age.
Split: one farm_field is held out (--test_field, default: last field alphabetically) and its
plots are split 50/50 into val (early stopping) and test; all other fields are used for training.
The best-val checkpoint is saved in Hugging Face format to --out_dir.
"""
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
import argparse
import json
import random
import time
import numpy as np
import pandas as pd
import torch
from sklearn.metrics import r2_score
from torch.utils.data import DataLoader
from transformers import AutoModel
from data.multimodal_dataset import MultimodalYieldDataset, compute_norm_stats, field_split, load_npz
from hf.auto import register_multimodal_autoclass
from hf.configuration_multimodal import MultimodalYieldConfig
def parse_args():
p = argparse.ArgumentParser()
p.add_argument("--data", required=True, help="embeddings .npz (see README)")
p.add_argument("--test_field", default=None,
help="farm_field held out for val/test (default: last field alphabetically); all others train")
p.add_argument("--cutoffs", default="4,8,12,16,20,22")
p.add_argument("--out_dir", default="checkpoints")
p.add_argument("--epochs", type=int, default=300)
p.add_argument("--patience", type=int, default=30)
p.add_argument("--batch_size", type=int, default=16)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--weight_decay", type=float, default=1e-2)
p.add_argument("--d_model", type=int, default=64)
p.add_argument("--num_layers", type=int, default=2)
p.add_argument("--pool", default="last", choices=["last", "mean"])
p.add_argument("--nhead", type=int, default=4)
p.add_argument("--dim_ff", type=int, default=128)
p.add_argument("--dropout", type=float, default=0.3)
p.add_argument("--input_dropout", type=float, default=0.2)
p.add_argument("--seed", type=int, default=1234)
return p.parse_args()
def set_seed(s):
random.seed(s)
np.random.seed(s)
torch.manual_seed(s)
torch.cuda.manual_seed_all(s)
def to_dev(batch, device):
return {k: (v.to(device) if torch.is_tensor(v) else v) for k, v in batch.items()}
def train_one_epoch(model, loader, opt, device, cutoffs):
model.train()
tot, n = 0.0, 0
for b in loader:
b = to_dev(b, device)
losses = [
model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t, labels=b["yield"]).loss
for t in cutoffs
]
loss = torch.stack(losses).mean()
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
opt.step()
tot += loss.item() * len(b["yield"])
n += len(b["yield"])
return tot / max(n, 1)
@torch.no_grad()
def evaluate(model, ds, device, cutoffs, batch_size=64):
model.eval()
loader = DataLoader(ds, batch_size=batch_size, shuffle=False)
res, rows = {}, []
for t in cutoffs:
yt, yp = [], []
for b in loader:
b = to_dev(b, device)
out = model(text=b["text"], image=b["image"], img_mask=b["img_mask"], img_age=b["img_age"], cutoff=t)
yt.append(b["yield"].cpu().numpy())
yp.append(out.predictions.cpu().numpy())
yt, yp = np.concatenate(yt), np.concatenate(yp)
err = yp - yt
res[int(t)] = {
"rmse": float(np.sqrt(np.mean(err ** 2))),
"mae": float(np.mean(np.abs(err))),
"bias": float(np.mean(err)),
"r2": float(r2_score(yt, yp)) if len(yt) > 1 else float("nan"),
}
for i in range(len(yt)):
rows.append({
"cutoff": int(t),
"farm_field": ds.meta["farm_field"][i],
"plot": int(ds.meta["plot"][i]),
"row_key": ds.meta["row_key"][i],
"y_true": float(yt[i]),
"y_pred": float(yp[i]),
"error": float(err[i]),
})
return res, rows
def print_table(title, res):
print(f"\n{title}\n{'Cutoff':>8}{'RMSE':>10}{'MAE':>10}{'Bias':>10}{'R2':>10}")
for t, m in res.items():
print(f"{t:>8}{m['rmse']:>10.3f}{m['mae']:>10.3f}{m['bias']:>10.3f}{m['r2']:>10.3f}")
def save_json(path, obj):
with open(path, "w") as f:
json.dump(obj, f, indent=2, default=str)
def main():
a = parse_args()
set_seed(a.seed)
register_multimodal_autoclass()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
out = Path(a.out_dir)
out.mkdir(parents=True, exist_ok=True)
save_json(out / "train_args.json", vars(a))
cutoffs = [int(c) for c in a.cutoffs.split(",")]
data = load_npz(a.data)
train_idx, val_idx, test_idx, train_fields, test_field = field_split(data, a.test_field, seed=a.seed)
print(f"device={device} train_fields={train_fields} test_field={test_field} "
f"train={len(train_idx)} val={len(val_idx)} test={len(test_idx)}")
stats = compute_norm_stats(data, train_idx)
cfg = MultimodalYieldConfig(
K=int(data["text"].shape[1]),
text_dim=int(data["text"].shape[2]),
image_dim=int(data["image"].shape[2]),
train_cutoffs=cutoffs,
eval_cutoffs=cutoffs,
d_model=a.d_model, nhead=a.nhead, num_layers=a.num_layers, dim_ff=a.dim_ff,
dropout=a.dropout, input_dropout=a.input_dropout, pool=a.pool,
**stats,
)
model = AutoModel.from_config(cfg).to(device)
print(f"params: {sum(p.numel() for p in model.parameters()):,}")
train_ds = MultimodalYieldDataset(data, train_idx)
loader = DataLoader(train_ds, batch_size=a.batch_size, shuffle=True, drop_last=len(train_ds) > a.batch_size)
opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=a.weight_decay)
val_ds = MultimodalYieldDataset(data, val_idx)
test_ds = MultimodalYieldDataset(data, test_idx)
best, bad, best_ep = float("inf"), 0, 0
t0 = time.time()
for ep in range(1, a.epochs + 1):
loss = train_one_epoch(model, loader, opt, device, cutoffs)
val_res, _ = evaluate(model, val_ds, device, cutoffs)
val_rmse = float(np.mean([m["rmse"] for m in val_res.values()]))
if val_rmse < best:
best, bad, best_ep = val_rmse, 0, ep
model.save_pretrained(out)
save_json(out / "metrics.json", {"epoch": ep, "val_rmse": val_rmse, "val_by_cutoff": val_res})
else:
bad += 1
if ep % 10 == 0 or bad == 0:
print(f"epoch {ep:03d} train_loss={loss:.4f} val_rmse={val_rmse:.3f} best={best:.3f}@{best_ep}")
if bad >= a.patience:
print(f"early stop at epoch {ep}")
break
print(f"training time {time.time() - t0:.1f}s, best epoch {best_ep}")
best_model = AutoModel.from_pretrained(out).to(device)
val_res, _ = evaluate(best_model, val_ds, device, cutoffs)
test_res, rows = evaluate(best_model, test_ds, device, cutoffs)
print_table(f"Val metrics ({test_field})", val_res)
print_table(f"Test metrics ({test_field})", test_res)
# Reference (log only): RMSE of predicting the training-set mean yield
yt = data["yield"][test_idx]
print(f"\nTrain-mean baseline test RMSE: {np.sqrt(np.mean((yt - stats['y_mean']) ** 2)):.3f}")
save_json(out / "test_metrics.json", {
"train_fields": train_fields, "test_field": test_field, "best_epoch": best_ep,
"n_train": len(train_idx), "n_val": len(val_idx), "n_test": len(test_idx),
"test_by_cutoff": test_res, "val_by_cutoff": val_res,
})
pd.DataFrame(rows).to_csv(out / "test_predictions.csv", index=False)
print(f"saved to {out}")
if __name__ == "__main__":
main()