""" evaluate.py — evaluate the SegFormer-B0 checkpoint on the BDAPPV test split. Mirrors train/evaluate.py: same SegmentationDataset transform, same per-batch metric aggregation, same CLI shape. The model is swapped for segmentation-models-pytorch's SegFormer-B0, and the input size defaults to 416 because SegFormer requires dimensions divisible by 32 (the reference 400 default raises a shape error). Usage: pip install segmentation-models-pytorch datasets python evaluate.py --provider ign --checkpoint segformer_ign_best.pth """ import argparse import torch import torch.nn as nn from torch.utils.data import DataLoader, Dataset from torchvision import transforms as T from datasets import load_dataset import segmentation_models_pytorch as smp IMG_MEAN = [0.485, 0.456, 0.406] IMG_STD = [0.229, 0.224, 0.225] class SegmentationDataset(Dataset): """Positives only (has_mask=True) -> (image [3,H,W], mask [1,H,W]).""" def __init__(self, hf_split, img_size: int = 416, augment: bool = False): self.data = hf_split.filter(lambda x: x["has_mask"]) self.augment = augment self.img_size = img_size self.img_tf = T.Compose([ T.Resize((img_size, img_size)), T.ToTensor(), T.Normalize(mean=IMG_MEAN, std=IMG_STD), ]) self.mask_tf = T.Compose([ T.Resize((img_size, img_size), interpolation=T.InterpolationMode.NEAREST), T.ToTensor(), ]) def __len__(self): return len(self.data) def __getitem__(self, idx): ex = self.data[idx] image = ex["image"].convert("RGB") mask = ex["mask"].convert("L") image = self.img_tf(image) mask = (self.mask_tf(mask) > 0.5).float() return image, mask def compute_seg_metrics(pred_logits, target, threshold=0.5): pred = (torch.sigmoid(pred_logits) > threshold).float() tp = (pred * target).sum().item() fp = (pred * (1 - target)).sum().item() fn = ((1 - pred) * target).sum().item() iou = tp / max(tp + fp + fn, 1e-6) prec = tp / max(tp + fp, 1e-6) rec = tp / max(tp + fn, 1e-6) f1 = 2 * prec * rec / max(prec + rec, 1e-6) return iou, f1, tp, fp, fn def eval_seg(args): device = ( torch.device("mps") if torch.backends.mps.is_available() else torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") ) ds = load_dataset("gabrielkasmi/bdappv", args.provider) test_ds = SegmentationDataset(ds["test"], img_size=args.img_size, augment=False) loader = DataLoader(test_ds, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) model = smp.Segformer( encoder_name=args.encoder, encoder_weights=None, in_channels=3, classes=1, ) state = torch.load(args.checkpoint, map_location="cpu", weights_only=False) if isinstance(state, dict) and "state_dict" in state: state = state["state_dict"] model.load_state_dict(state) model = model.to(device) model.eval() total_iou, total_f1 = 0.0, 0.0 tp_all = fp_all = fn_all = 0.0 with torch.no_grad(): for images, masks in loader: images, masks = images.to(device), masks.to(device) logits = model(images) if logits.shape[-2:] != masks.shape[-2:]: logits = nn.functional.interpolate( logits, size=masks.shape[-2:], mode="bilinear", align_corners=False) iou, f1, tp, fp, fn = compute_seg_metrics( logits, masks, threshold=args.threshold) total_iou += iou total_f1 += f1 tp_all += tp fp_all += fp fn_all += fn n = len(loader) global_iou = tp_all / max(tp_all + fp_all + fn_all, 1e-6) global_f1 = 2 * tp_all / max(2 * tp_all + fp_all + fn_all, 1e-6) # Keep the CLI output portable on Windows consoles using the cp1252 code page. print("\n== Segmentation results ==") print(f" Provider : {args.provider}") print(f" Checkpoint : {args.checkpoint}") print(f" Test images: {len(test_ds):,}") print(f" IoU : {total_iou / n:.4f}") print(f" F1 : {total_f1 / n:.4f}") print(f"\n (reference protocol: mean of per-batch IoU/F1, " f"batch_size={args.batch_size})") print(f" Pixel-aggregated over the split: " f"IoU {global_iou:.4f}, F1 {global_f1:.4f}") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--provider", required=True, choices=["google", "ign"]) parser.add_argument("--checkpoint", required=True) parser.add_argument("--encoder", default="mit_b0") parser.add_argument("--batch_size", type=int, default=8) parser.add_argument("--img_size", type=int, default=416) parser.add_argument("--num_workers", type=int, default=0) parser.add_argument("--threshold", type=float, default=0.5) eval_seg(parser.parse_args())