Spaces:
Sleeping
Sleeping
Download plant_classifier/evaluation.py from ZZW45/image-classifier: direct link, hf CLI and curl.
- Browser
- Download file 7.19 kB
-
https://huggingface.co/spaces/ZZW45/image-classifier/resolve/main/plant_classifier/evaluation.py
- Command line
-
hf download hf://spaces/ZZW45/image-classifier/plant_classifier/evaluation.py
-
curl -L -o evaluation.py https://huggingface.co/spaces/ZZW45/image-classifier/resolve/main/plant_classifier/evaluation.py
7.19 kB
| """Evaluate a selected checkpoint on the fixed, previously unused test split.""" | |
| from __future__ import annotations | |
| import csv | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| from sklearn.metrics import classification_report, confusion_matrix, f1_score | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from .checkpoint import load_checkpoint | |
| from .data import ImageDataset, load_prepared_data | |
| from .inference import prediction_transform, resolve_device, write_json | |
| def _validate_dataset(checkpoint: dict[str, Any], manifest: dict[str, Any]) -> None: | |
| if checkpoint["class_names"] != manifest["class_names"]: | |
| raise ValueError("Dataset class mapping does not match the checkpoint.") | |
| for key in ("dataset_id", "revision", "seed"): | |
| if checkpoint["data"].get(key) != manifest.get(key): | |
| raise ValueError(f"Dataset {key} does not match the checkpoint.") | |
| saved_hash = checkpoint["data"].get("split_hash") | |
| actual_hash = hashlib.sha256( | |
| json.dumps(manifest["splits"], sort_keys=True, separators=(",", ":")).encode("utf-8") | |
| ).hexdigest() | |
| if actual_hash != saved_hash: | |
| raise ValueError("Dataset split indices do not match the checkpoint.") | |
| def _save_reports( | |
| destination: Path, | |
| class_names: list[str], | |
| targets: np.ndarray, | |
| predictions: np.ndarray, | |
| ) -> None: | |
| labels = list(range(len(class_names))) | |
| report = classification_report( | |
| targets, predictions, labels=labels, target_names=class_names, | |
| output_dict=True, zero_division=0, | |
| ) | |
| write_json(destination / "classification_report.json", report) | |
| with (destination / "classification_report.csv").open("w", newline="", encoding="utf-8") as stream: | |
| writer = csv.writer(stream) | |
| writer.writerow(["class_id", "class_name", "precision", "recall", "f1_score", "support"]) | |
| for index, name in enumerate(class_names): | |
| row = report[name] | |
| writer.writerow([index, name, row["precision"], row["recall"], row["f1-score"], int(row["support"])]) | |
| matrix = confusion_matrix(targets, predictions, labels=labels) | |
| with (destination / "confusion_matrix.csv").open("w", newline="", encoding="utf-8") as stream: | |
| writer = csv.writer(stream) | |
| writer.writerow(["true_class / predicted_class", *class_names]) | |
| for name, counts in zip(class_names, matrix): | |
| writer.writerow([name, *counts.tolist()]) | |
| row_counts = matrix.sum(axis=1, keepdims=True) | |
| normalized = np.divide(matrix, row_counts, out=np.zeros_like(matrix, dtype=float), where=row_counts != 0) | |
| side = max(8, 0.43 * len(class_names)) | |
| figure, axis = plt.subplots(figsize=(side + 4, side)) | |
| display = axis.imshow(normalized, interpolation="nearest", cmap="Blues", vmin=0, vmax=1) | |
| figure.colorbar(display, ax=axis, fraction=0.04, pad=0.02, label="Fraction of true class") | |
| axis.set( | |
| xticks=labels, yticks=labels, xticklabels=class_names, yticklabels=class_names, | |
| xlabel="Predicted class", ylabel="True class", title="Test confusion matrix (row normalized)", | |
| ) | |
| plt.setp(axis.get_xticklabels(), rotation=90, fontsize=8) | |
| plt.setp(axis.get_yticklabels(), fontsize=8) | |
| if len(class_names) <= 10: | |
| for row in labels: | |
| for column in labels: | |
| axis.text(column, row, str(matrix[row, column]), ha="center", va="center", | |
| color="white" if normalized[row, column] > 0.5 else "black") | |
| figure.tight_layout() | |
| figure.savefig(destination / "confusion_matrix.png", dpi=160, bbox_inches="tight") | |
| plt.close(figure) | |
| def evaluate( | |
| checkpoint_path: str, | |
| cache_dir: str, | |
| data_dir: str, | |
| output_dir: str, | |
| device: str = "auto", | |
| batch_size: int = 128, | |
| num_workers: int = 4, | |
| ) -> dict[str, Any]: | |
| """Report metrics for every class, using exactly the checkpoint's data split.""" | |
| if batch_size < 1 or num_workers < 0: | |
| raise ValueError("batch_size must be positive and num_workers must be non-negative.") | |
| selected_device = resolve_device(device) | |
| model, checkpoint = load_checkpoint(checkpoint_path, selected_device) | |
| raw, manifest = load_prepared_data(cache_dir, data_dir) | |
| _validate_dataset(checkpoint, manifest) | |
| class_names = checkpoint["class_names"] | |
| test_indices = manifest["splits"]["test"] | |
| if not test_indices: | |
| raise ValueError("The fixed test split is empty.") | |
| dataset = ImageDataset(raw, test_indices, prediction_transform(checkpoint)) | |
| loader = DataLoader( | |
| dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, | |
| pin_memory=selected_device.type == "cuda", | |
| ) | |
| criterion = torch.nn.CrossEntropyLoss(reduction="sum") | |
| total_loss = 0.0 | |
| top5_correct = 0 | |
| target_batches: list[np.ndarray] = [] | |
| prediction_batches: list[np.ndarray] = [] | |
| model.eval() | |
| with torch.inference_mode(): | |
| for images, targets in loader: | |
| images = images.to(selected_device, non_blocking=True) | |
| targets = targets.to(selected_device, non_blocking=True) | |
| logits = model(images).float() | |
| if not torch.isfinite(logits).all(): | |
| raise ValueError("Model produced non-finite logits during evaluation.") | |
| total_loss += float(criterion(logits, targets).item()) | |
| top5_correct += int(logits.topk(min(5, len(class_names)), dim=1).indices.eq(targets[:, None]).any(dim=1).sum().item()) | |
| target_batches.append(targets.cpu().numpy()) | |
| prediction_batches.append(logits.argmax(dim=1).cpu().numpy()) | |
| targets = np.concatenate(target_batches) | |
| predictions = np.concatenate(prediction_batches) | |
| all_labels = np.asarray(raw["label"], dtype=np.int64) | |
| training_labels = all_labels[manifest["splits"]["train"]] | |
| if not len(training_labels): | |
| raise ValueError("The training split is empty; cannot define a majority-class baseline.") | |
| majority_class = int(np.bincount(training_labels, minlength=len(class_names)).argmax()) | |
| metrics: dict[str, Any] = { | |
| "loss": total_loss / len(targets), | |
| "top1_accuracy": float(np.mean(targets == predictions)), | |
| "top5_accuracy": top5_correct / len(targets), | |
| "macro_f1": float(f1_score(targets, predictions, labels=list(range(len(class_names))), average="macro", zero_division=0)), | |
| "majority_baseline_accuracy": float(np.mean(targets == majority_class)), | |
| "majority_baseline_class": class_names[majority_class], | |
| "num_samples": int(len(targets)), | |
| "num_classes": len(class_names), | |
| "checkpoint": str(Path(checkpoint_path).resolve()), | |
| "checkpoint_epoch": checkpoint.get("epoch"), | |
| "data": checkpoint["data"], | |
| "split": "test", | |
| } | |
| destination = Path(output_dir) | |
| destination.mkdir(parents=True, exist_ok=True) | |
| _save_reports(destination, class_names, targets, predictions) | |
| # Write the completion marker last, so interrupted report generation is retried. | |
| write_json(destination / "metrics.json", metrics) | |
| return metrics | |