bdappv-models / segformer-b0 /evaluate.py
gabrielkasmi's picture masteroshi006's picture
Add SegFormer-B0 (IGN) checkpoint, config and metrics (#1)
11e732c
Raw History Blame Contribute Delete
5.02 kB
"""
evaluate.py — evaluate the SegFormer-B0 checkpoint on the BDAPPV test split.
Mirrors train/evaluate.py: same SegmentationDataset transform, same per-batch
metric aggregation, same CLI shape. The model is swapped for
segmentation-models-pytorch's SegFormer-B0, and the input size defaults to 416
because SegFormer requires dimensions divisible by 32 (the reference 400
default raises a shape error).
Usage:
pip install segmentation-models-pytorch datasets
python evaluate.py --provider ign --checkpoint segformer_ign_best.pth
"""
import argparse
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms as T
from datasets import load_dataset
import segmentation_models_pytorch as smp
IMG_MEAN = [0.485, 0.456, 0.406]
IMG_STD = [0.229, 0.224, 0.225]
class SegmentationDataset(Dataset):
"""Positives only (has_mask=True) -> (image [3,H,W], mask [1,H,W])."""
def __init__(self, hf_split, img_size: int = 416, augment: bool = False):
self.data = hf_split.filter(lambda x: x["has_mask"])
self.augment = augment
self.img_size = img_size
self.img_tf = T.Compose([
T.Resize((img_size, img_size)),
T.ToTensor(),
T.Normalize(mean=IMG_MEAN, std=IMG_STD),
])
self.mask_tf = T.Compose([
T.Resize((img_size, img_size), interpolation=T.InterpolationMode.NEAREST),
T.ToTensor(),
])
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
ex = self.data[idx]
image = ex["image"].convert("RGB")
mask = ex["mask"].convert("L")
image = self.img_tf(image)
mask = (self.mask_tf(mask) > 0.5).float()
return image, mask
def compute_seg_metrics(pred_logits, target, threshold=0.5):
pred = (torch.sigmoid(pred_logits) > threshold).float()
tp = (pred * target).sum().item()
fp = (pred * (1 - target)).sum().item()
fn = ((1 - pred) * target).sum().item()
iou = tp / max(tp + fp + fn, 1e-6)
prec = tp / max(tp + fp, 1e-6)
rec = tp / max(tp + fn, 1e-6)
f1 = 2 * prec * rec / max(prec + rec, 1e-6)
return iou, f1, tp, fp, fn
def eval_seg(args):
device = (
torch.device("mps") if torch.backends.mps.is_available() else
torch.device("cuda") if torch.cuda.is_available() else
torch.device("cpu")
)
ds = load_dataset("gabrielkasmi/bdappv", args.provider)
test_ds = SegmentationDataset(ds["test"], img_size=args.img_size, augment=False)
loader = DataLoader(test_ds, batch_size=args.batch_size, shuffle=False,
num_workers=args.num_workers)
model = smp.Segformer(
encoder_name=args.encoder,
encoder_weights=None,
in_channels=3,
classes=1,
)
state = torch.load(args.checkpoint, map_location="cpu", weights_only=False)
if isinstance(state, dict) and "state_dict" in state:
state = state["state_dict"]
model.load_state_dict(state)
model = model.to(device)
model.eval()
total_iou, total_f1 = 0.0, 0.0
tp_all = fp_all = fn_all = 0.0
with torch.no_grad():
for images, masks in loader:
images, masks = images.to(device), masks.to(device)
logits = model(images)
if logits.shape[-2:] != masks.shape[-2:]:
logits = nn.functional.interpolate(
logits, size=masks.shape[-2:], mode="bilinear", align_corners=False)
iou, f1, tp, fp, fn = compute_seg_metrics(
logits, masks, threshold=args.threshold)
total_iou += iou
total_f1 += f1
tp_all += tp
fp_all += fp
fn_all += fn
n = len(loader)
global_iou = tp_all / max(tp_all + fp_all + fn_all, 1e-6)
global_f1 = 2 * tp_all / max(2 * tp_all + fp_all + fn_all, 1e-6)
# Keep the CLI output portable on Windows consoles using the cp1252 code page.
print("\n== Segmentation results ==")
print(f" Provider : {args.provider}")
print(f" Checkpoint : {args.checkpoint}")
print(f" Test images: {len(test_ds):,}")
print(f" IoU : {total_iou / n:.4f}")
print(f" F1 : {total_f1 / n:.4f}")
print(f"\n (reference protocol: mean of per-batch IoU/F1, "
f"batch_size={args.batch_size})")
print(f" Pixel-aggregated over the split: "
f"IoU {global_iou:.4f}, F1 {global_f1:.4f}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--provider", required=True, choices=["google", "ign"])
parser.add_argument("--checkpoint", required=True)
parser.add_argument("--encoder", default="mit_b0")
parser.add_argument("--batch_size", type=int, default=8)
parser.add_argument("--img_size", type=int, default=416)
parser.add_argument("--num_workers", type=int, default=0)
parser.add_argument("--threshold", type=float, default=0.5)
eval_seg(parser.parse_args())