ZZW45's picture
Deploy PlantVillage Gradio classifier with CPU and ZeroGPU support
cc0465b verified
Raw History Blame Contribute Delete
7.19 kB
"""Evaluate a selected checkpoint on the fixed, previously unused test split."""
from __future__ import annotations
import csv
import hashlib
import json
from pathlib import Path
from typing import Any
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix, f1_score
import torch
from torch.utils.data import DataLoader
from .checkpoint import load_checkpoint
from .data import ImageDataset, load_prepared_data
from .inference import prediction_transform, resolve_device, write_json
def _validate_dataset(checkpoint: dict[str, Any], manifest: dict[str, Any]) -> None:
if checkpoint["class_names"] != manifest["class_names"]:
raise ValueError("Dataset class mapping does not match the checkpoint.")
for key in ("dataset_id", "revision", "seed"):
if checkpoint["data"].get(key) != manifest.get(key):
raise ValueError(f"Dataset {key} does not match the checkpoint.")
saved_hash = checkpoint["data"].get("split_hash")
actual_hash = hashlib.sha256(
json.dumps(manifest["splits"], sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if actual_hash != saved_hash:
raise ValueError("Dataset split indices do not match the checkpoint.")
def _save_reports(
destination: Path,
class_names: list[str],
targets: np.ndarray,
predictions: np.ndarray,
) -> None:
labels = list(range(len(class_names)))
report = classification_report(
targets, predictions, labels=labels, target_names=class_names,
output_dict=True, zero_division=0,
)
write_json(destination / "classification_report.json", report)
with (destination / "classification_report.csv").open("w", newline="", encoding="utf-8") as stream:
writer = csv.writer(stream)
writer.writerow(["class_id", "class_name", "precision", "recall", "f1_score", "support"])
for index, name in enumerate(class_names):
row = report[name]
writer.writerow([index, name, row["precision"], row["recall"], row["f1-score"], int(row["support"])])
matrix = confusion_matrix(targets, predictions, labels=labels)
with (destination / "confusion_matrix.csv").open("w", newline="", encoding="utf-8") as stream:
writer = csv.writer(stream)
writer.writerow(["true_class / predicted_class", *class_names])
for name, counts in zip(class_names, matrix):
writer.writerow([name, *counts.tolist()])
row_counts = matrix.sum(axis=1, keepdims=True)
normalized = np.divide(matrix, row_counts, out=np.zeros_like(matrix, dtype=float), where=row_counts != 0)
side = max(8, 0.43 * len(class_names))
figure, axis = plt.subplots(figsize=(side + 4, side))
display = axis.imshow(normalized, interpolation="nearest", cmap="Blues", vmin=0, vmax=1)
figure.colorbar(display, ax=axis, fraction=0.04, pad=0.02, label="Fraction of true class")
axis.set(
xticks=labels, yticks=labels, xticklabels=class_names, yticklabels=class_names,
xlabel="Predicted class", ylabel="True class", title="Test confusion matrix (row normalized)",
)
plt.setp(axis.get_xticklabels(), rotation=90, fontsize=8)
plt.setp(axis.get_yticklabels(), fontsize=8)
if len(class_names) <= 10:
for row in labels:
for column in labels:
axis.text(column, row, str(matrix[row, column]), ha="center", va="center",
color="white" if normalized[row, column] > 0.5 else "black")
figure.tight_layout()
figure.savefig(destination / "confusion_matrix.png", dpi=160, bbox_inches="tight")
plt.close(figure)
def evaluate(
checkpoint_path: str,
cache_dir: str,
data_dir: str,
output_dir: str,
device: str = "auto",
batch_size: int = 128,
num_workers: int = 4,
) -> dict[str, Any]:
"""Report metrics for every class, using exactly the checkpoint's data split."""
if batch_size < 1 or num_workers < 0:
raise ValueError("batch_size must be positive and num_workers must be non-negative.")
selected_device = resolve_device(device)
model, checkpoint = load_checkpoint(checkpoint_path, selected_device)
raw, manifest = load_prepared_data(cache_dir, data_dir)
_validate_dataset(checkpoint, manifest)
class_names = checkpoint["class_names"]
test_indices = manifest["splits"]["test"]
if not test_indices:
raise ValueError("The fixed test split is empty.")
dataset = ImageDataset(raw, test_indices, prediction_transform(checkpoint))
loader = DataLoader(
dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers,
pin_memory=selected_device.type == "cuda",
)
criterion = torch.nn.CrossEntropyLoss(reduction="sum")
total_loss = 0.0
top5_correct = 0
target_batches: list[np.ndarray] = []
prediction_batches: list[np.ndarray] = []
model.eval()
with torch.inference_mode():
for images, targets in loader:
images = images.to(selected_device, non_blocking=True)
targets = targets.to(selected_device, non_blocking=True)
logits = model(images).float()
if not torch.isfinite(logits).all():
raise ValueError("Model produced non-finite logits during evaluation.")
total_loss += float(criterion(logits, targets).item())
top5_correct += int(logits.topk(min(5, len(class_names)), dim=1).indices.eq(targets[:, None]).any(dim=1).sum().item())
target_batches.append(targets.cpu().numpy())
prediction_batches.append(logits.argmax(dim=1).cpu().numpy())
targets = np.concatenate(target_batches)
predictions = np.concatenate(prediction_batches)
all_labels = np.asarray(raw["label"], dtype=np.int64)
training_labels = all_labels[manifest["splits"]["train"]]
if not len(training_labels):
raise ValueError("The training split is empty; cannot define a majority-class baseline.")
majority_class = int(np.bincount(training_labels, minlength=len(class_names)).argmax())
metrics: dict[str, Any] = {
"loss": total_loss / len(targets),
"top1_accuracy": float(np.mean(targets == predictions)),
"top5_accuracy": top5_correct / len(targets),
"macro_f1": float(f1_score(targets, predictions, labels=list(range(len(class_names))), average="macro", zero_division=0)),
"majority_baseline_accuracy": float(np.mean(targets == majority_class)),
"majority_baseline_class": class_names[majority_class],
"num_samples": int(len(targets)),
"num_classes": len(class_names),
"checkpoint": str(Path(checkpoint_path).resolve()),
"checkpoint_epoch": checkpoint.get("epoch"),
"data": checkpoint["data"],
"split": "test",
}
destination = Path(output_dir)
destination.mkdir(parents=True, exist_ok=True)
_save_reports(destination, class_names, targets, predictions)
# Write the completion marker last, so interrupted report generation is retried.
write_json(destination / "metrics.json", metrics)
return metrics