"""Evaluate a selected checkpoint on the fixed, previously unused test split.""" from __future__ import annotations import csv import hashlib import json from pathlib import Path from typing import Any import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np from sklearn.metrics import classification_report, confusion_matrix, f1_score import torch from torch.utils.data import DataLoader from .checkpoint import load_checkpoint from .data import ImageDataset, load_prepared_data from .inference import prediction_transform, resolve_device, write_json def _validate_dataset(checkpoint: dict[str, Any], manifest: dict[str, Any]) -> None: if checkpoint["class_names"] != manifest["class_names"]: raise ValueError("Dataset class mapping does not match the checkpoint.") for key in ("dataset_id", "revision", "seed"): if checkpoint["data"].get(key) != manifest.get(key): raise ValueError(f"Dataset {key} does not match the checkpoint.") saved_hash = checkpoint["data"].get("split_hash") actual_hash = hashlib.sha256( json.dumps(manifest["splits"], sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() if actual_hash != saved_hash: raise ValueError("Dataset split indices do not match the checkpoint.") def _save_reports( destination: Path, class_names: list[str], targets: np.ndarray, predictions: np.ndarray, ) -> None: labels = list(range(len(class_names))) report = classification_report( targets, predictions, labels=labels, target_names=class_names, output_dict=True, zero_division=0, ) write_json(destination / "classification_report.json", report) with (destination / "classification_report.csv").open("w", newline="", encoding="utf-8") as stream: writer = csv.writer(stream) writer.writerow(["class_id", "class_name", "precision", "recall", "f1_score", "support"]) for index, name in enumerate(class_names): row = report[name] writer.writerow([index, name, row["precision"], row["recall"], row["f1-score"], int(row["support"])]) matrix = confusion_matrix(targets, predictions, labels=labels) with (destination / "confusion_matrix.csv").open("w", newline="", encoding="utf-8") as stream: writer = csv.writer(stream) writer.writerow(["true_class / predicted_class", *class_names]) for name, counts in zip(class_names, matrix): writer.writerow([name, *counts.tolist()]) row_counts = matrix.sum(axis=1, keepdims=True) normalized = np.divide(matrix, row_counts, out=np.zeros_like(matrix, dtype=float), where=row_counts != 0) side = max(8, 0.43 * len(class_names)) figure, axis = plt.subplots(figsize=(side + 4, side)) display = axis.imshow(normalized, interpolation="nearest", cmap="Blues", vmin=0, vmax=1) figure.colorbar(display, ax=axis, fraction=0.04, pad=0.02, label="Fraction of true class") axis.set( xticks=labels, yticks=labels, xticklabels=class_names, yticklabels=class_names, xlabel="Predicted class", ylabel="True class", title="Test confusion matrix (row normalized)", ) plt.setp(axis.get_xticklabels(), rotation=90, fontsize=8) plt.setp(axis.get_yticklabels(), fontsize=8) if len(class_names) <= 10: for row in labels: for column in labels: axis.text(column, row, str(matrix[row, column]), ha="center", va="center", color="white" if normalized[row, column] > 0.5 else "black") figure.tight_layout() figure.savefig(destination / "confusion_matrix.png", dpi=160, bbox_inches="tight") plt.close(figure) def evaluate( checkpoint_path: str, cache_dir: str, data_dir: str, output_dir: str, device: str = "auto", batch_size: int = 128, num_workers: int = 4, ) -> dict[str, Any]: """Report metrics for every class, using exactly the checkpoint's data split.""" if batch_size < 1 or num_workers < 0: raise ValueError("batch_size must be positive and num_workers must be non-negative.") selected_device = resolve_device(device) model, checkpoint = load_checkpoint(checkpoint_path, selected_device) raw, manifest = load_prepared_data(cache_dir, data_dir) _validate_dataset(checkpoint, manifest) class_names = checkpoint["class_names"] test_indices = manifest["splits"]["test"] if not test_indices: raise ValueError("The fixed test split is empty.") dataset = ImageDataset(raw, test_indices, prediction_transform(checkpoint)) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=selected_device.type == "cuda", ) criterion = torch.nn.CrossEntropyLoss(reduction="sum") total_loss = 0.0 top5_correct = 0 target_batches: list[np.ndarray] = [] prediction_batches: list[np.ndarray] = [] model.eval() with torch.inference_mode(): for images, targets in loader: images = images.to(selected_device, non_blocking=True) targets = targets.to(selected_device, non_blocking=True) logits = model(images).float() if not torch.isfinite(logits).all(): raise ValueError("Model produced non-finite logits during evaluation.") total_loss += float(criterion(logits, targets).item()) top5_correct += int(logits.topk(min(5, len(class_names)), dim=1).indices.eq(targets[:, None]).any(dim=1).sum().item()) target_batches.append(targets.cpu().numpy()) prediction_batches.append(logits.argmax(dim=1).cpu().numpy()) targets = np.concatenate(target_batches) predictions = np.concatenate(prediction_batches) all_labels = np.asarray(raw["label"], dtype=np.int64) training_labels = all_labels[manifest["splits"]["train"]] if not len(training_labels): raise ValueError("The training split is empty; cannot define a majority-class baseline.") majority_class = int(np.bincount(training_labels, minlength=len(class_names)).argmax()) metrics: dict[str, Any] = { "loss": total_loss / len(targets), "top1_accuracy": float(np.mean(targets == predictions)), "top5_accuracy": top5_correct / len(targets), "macro_f1": float(f1_score(targets, predictions, labels=list(range(len(class_names))), average="macro", zero_division=0)), "majority_baseline_accuracy": float(np.mean(targets == majority_class)), "majority_baseline_class": class_names[majority_class], "num_samples": int(len(targets)), "num_classes": len(class_names), "checkpoint": str(Path(checkpoint_path).resolve()), "checkpoint_epoch": checkpoint.get("epoch"), "data": checkpoint["data"], "split": "test", } destination = Path(output_dir) destination.mkdir(parents=True, exist_ok=True) _save_reports(destination, class_names, targets, predictions) # Write the completion marker last, so interrupted report generation is retried. write_json(destination / "metrics.json", metrics) return metrics