Celyrix-1.0-Vision / src /evaluate.py
mikexsav's picture
Release CELYRIX v1.0.0
17124c2
Raw History Blame Contribute Delete
4.73 kB
"""Evaluate frozen validation-selected model on held-out test data."""
import argparse
import json
from pathlib import Path
import numpy as np
from src.common import CLASSES, configure
from src.dataset import load_data, pipeline
def main():
p = argparse.ArgumentParser()
p.add_argument("--model-dir", default="artifacts/release")
args = p.parse_args()
out = Path(args.model_dir)
cfg = json.loads((out / "training_config.json").read_text())
configure(cfg["seed"])
import matplotlib
import tensorflow as tf
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
matplotlib.use("Agg")
import matplotlib.pyplot as plt
splits, provenance = load_data(cfg["data_dir"], cfg["seed"])
saved_provenance = json.loads((out / "provenance.json").read_text())
if (
provenance["archive_sha256"] != saved_provenance["archive_sha256"]
or provenance["samples"] != saved_provenance["samples"]
):
raise ValueError("Evaluation provenance differs from training")
sets = [
{r["pixel_sha256"] for r in provenance["samples"][s]}
for s in ("train", "validation", "test")
]
assert all(not sets[a].intersection(sets[b]) for a, b in ((0, 1), (0, 2), (1, 2)))
model = tf.keras.models.load_model(
out / "model.keras", compile=False, safe_mode=True
)
x, y = splits["test"]
def predict(images):
return model.predict(
pipeline(images, y, cfg["batch_size"]).map(lambda a, b: a), verbose=0
)
probs = predict(x)
pred = probs.argmax(1)
if not np.isfinite(probs).all() or not np.allclose(probs.sum(1), 1, atol=1e-5):
raise ValueError("Invalid probability distribution")
report = classification_report(
y,
pred,
labels=list(range(5)),
target_names=CLASSES,
output_dict=True,
zero_division=0,
)
matrix = confusion_matrix(y, pred, labels=list(range(5)))
robustness = {}
for name, transformed in [
("rotate_90", np.rot90(x, axes=(1, 2))),
("horizontal_flip", x[:, :, ::-1]),
("brightness_0.8", np.clip(x.astype(float) * 0.8, 0, 255).astype("uint8")),
]:
robustness[name] = float(accuracy_score(y, predict(transformed).argmax(1)))
metrics = {
"test_samples": len(y),
"accuracy": float(accuracy_score(y, pred)),
"macro_precision": report["macro avg"]["precision"],
"macro_recall": report["macro avg"]["recall"],
"macro_f1": report["macro avg"]["f1-score"],
"per_class": report,
"confusion_matrix": matrix.tolist(),
"robustness_accuracy": robustness,
"exact_pixel_overlap": False,
"spatial_independence_verified": False,
}
(out / "metrics.json").write_text(json.dumps(metrics, indent=2))
fig, ax = plt.subplots(figsize=(7, 6))
ax.imshow(matrix, cmap="Blues")
ax.set(
xticks=range(5),
yticks=range(5),
xticklabels=CLASSES,
yticklabels=CLASSES,
xlabel="Predicted",
ylabel="True",
title="CELYRIX · held-out EuroSAT test set",
)
plt.setp(ax.get_xticklabels(), rotation=30, ha="right")
for i in range(5):
for j in range(5):
ax.text(
j,
i,
str(matrix[i, j]),
ha="center",
va="center",
color="white" if matrix[i, j] > matrix.max() / 2 else "black",
)
fig.tight_layout()
fig.savefig(out / "confusion_matrix.png", dpi=160)
plt.close(fig)
hist = json.loads((out / "history.json").read_text())
fig, axs = plt.subplots(1, 2, figsize=(10, 4))
for ax, key in zip(axs, ("loss", "accuracy")):
for prefix in ("", "val_"):
ax.plot(
range(1, len(hist[key]) + 1), hist[prefix + key], label=prefix + key
)
ax.set(xlabel="Epoch", ylabel=key)
ax.legend()
fig.tight_layout()
fig.savefig(out / "training_history.png", dpi=160)
plt.close(fig)
failures = []
for i in np.flatnonzero(pred != y)[:30]:
failures.append(
{
"filename": provenance["samples"]["test"][i]["filename"],
"true": CLASSES[y[i]],
"predicted": CLASSES[pred[i]],
"max_softmax": float(probs[i].max()),
}
)
(out / "failure_cases.json").write_text(json.dumps(failures, indent=2))
np.savez_compressed(out / "test_predictions.npz", labels=y, probabilities=probs)
print(json.dumps({k: v for k, v in metrics.items() if k != "per_class"}, indent=2))
if __name__ == "__main__":
main()