"""Evaluate frozen validation-selected model on held-out test data.""" import argparse import json from pathlib import Path import numpy as np from src.common import CLASSES, configure from src.dataset import load_data, pipeline def main(): p = argparse.ArgumentParser() p.add_argument("--model-dir", default="artifacts/release") args = p.parse_args() out = Path(args.model_dir) cfg = json.loads((out / "training_config.json").read_text()) configure(cfg["seed"]) import matplotlib import tensorflow as tf from sklearn.metrics import accuracy_score, classification_report, confusion_matrix matplotlib.use("Agg") import matplotlib.pyplot as plt splits, provenance = load_data(cfg["data_dir"], cfg["seed"]) saved_provenance = json.loads((out / "provenance.json").read_text()) if ( provenance["archive_sha256"] != saved_provenance["archive_sha256"] or provenance["samples"] != saved_provenance["samples"] ): raise ValueError("Evaluation provenance differs from training") sets = [ {r["pixel_sha256"] for r in provenance["samples"][s]} for s in ("train", "validation", "test") ] assert all(not sets[a].intersection(sets[b]) for a, b in ((0, 1), (0, 2), (1, 2))) model = tf.keras.models.load_model( out / "model.keras", compile=False, safe_mode=True ) x, y = splits["test"] def predict(images): return model.predict( pipeline(images, y, cfg["batch_size"]).map(lambda a, b: a), verbose=0 ) probs = predict(x) pred = probs.argmax(1) if not np.isfinite(probs).all() or not np.allclose(probs.sum(1), 1, atol=1e-5): raise ValueError("Invalid probability distribution") report = classification_report( y, pred, labels=list(range(5)), target_names=CLASSES, output_dict=True, zero_division=0, ) matrix = confusion_matrix(y, pred, labels=list(range(5))) robustness = {} for name, transformed in [ ("rotate_90", np.rot90(x, axes=(1, 2))), ("horizontal_flip", x[:, :, ::-1]), ("brightness_0.8", np.clip(x.astype(float) * 0.8, 0, 255).astype("uint8")), ]: robustness[name] = float(accuracy_score(y, predict(transformed).argmax(1))) metrics = { "test_samples": len(y), "accuracy": float(accuracy_score(y, pred)), "macro_precision": report["macro avg"]["precision"], "macro_recall": report["macro avg"]["recall"], "macro_f1": report["macro avg"]["f1-score"], "per_class": report, "confusion_matrix": matrix.tolist(), "robustness_accuracy": robustness, "exact_pixel_overlap": False, "spatial_independence_verified": False, } (out / "metrics.json").write_text(json.dumps(metrics, indent=2)) fig, ax = plt.subplots(figsize=(7, 6)) ax.imshow(matrix, cmap="Blues") ax.set( xticks=range(5), yticks=range(5), xticklabels=CLASSES, yticklabels=CLASSES, xlabel="Predicted", ylabel="True", title="CELYRIX ยท held-out EuroSAT test set", ) plt.setp(ax.get_xticklabels(), rotation=30, ha="right") for i in range(5): for j in range(5): ax.text( j, i, str(matrix[i, j]), ha="center", va="center", color="white" if matrix[i, j] > matrix.max() / 2 else "black", ) fig.tight_layout() fig.savefig(out / "confusion_matrix.png", dpi=160) plt.close(fig) hist = json.loads((out / "history.json").read_text()) fig, axs = plt.subplots(1, 2, figsize=(10, 4)) for ax, key in zip(axs, ("loss", "accuracy")): for prefix in ("", "val_"): ax.plot( range(1, len(hist[key]) + 1), hist[prefix + key], label=prefix + key ) ax.set(xlabel="Epoch", ylabel=key) ax.legend() fig.tight_layout() fig.savefig(out / "training_history.png", dpi=160) plt.close(fig) failures = [] for i in np.flatnonzero(pred != y)[:30]: failures.append( { "filename": provenance["samples"]["test"][i]["filename"], "true": CLASSES[y[i]], "predicted": CLASSES[pred[i]], "max_softmax": float(probs[i].max()), } ) (out / "failure_cases.json").write_text(json.dumps(failures, indent=2)) np.savez_compressed(out / "test_predictions.npz", labels=y, probabilities=probs) print(json.dumps({k: v for k, v in metrics.items() if k != "per_class"}, indent=2)) if __name__ == "__main__": main()