File size: 4,725 Bytes
17124c2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""Evaluate frozen validation-selected model on held-out test data."""

import argparse
import json
from pathlib import Path

import numpy as np

from src.common import CLASSES, configure
from src.dataset import load_data, pipeline


def main():
    p = argparse.ArgumentParser()
    p.add_argument("--model-dir", default="artifacts/release")
    args = p.parse_args()
    out = Path(args.model_dir)
    cfg = json.loads((out / "training_config.json").read_text())
    configure(cfg["seed"])
    import matplotlib
    import tensorflow as tf
    from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt

    splits, provenance = load_data(cfg["data_dir"], cfg["seed"])
    saved_provenance = json.loads((out / "provenance.json").read_text())
    if (
        provenance["archive_sha256"] != saved_provenance["archive_sha256"]
        or provenance["samples"] != saved_provenance["samples"]
    ):
        raise ValueError("Evaluation provenance differs from training")
    sets = [
        {r["pixel_sha256"] for r in provenance["samples"][s]}
        for s in ("train", "validation", "test")
    ]
    assert all(not sets[a].intersection(sets[b]) for a, b in ((0, 1), (0, 2), (1, 2)))
    model = tf.keras.models.load_model(
        out / "model.keras", compile=False, safe_mode=True
    )
    x, y = splits["test"]

    def predict(images):
        return model.predict(
            pipeline(images, y, cfg["batch_size"]).map(lambda a, b: a), verbose=0
        )

    probs = predict(x)
    pred = probs.argmax(1)
    if not np.isfinite(probs).all() or not np.allclose(probs.sum(1), 1, atol=1e-5):
        raise ValueError("Invalid probability distribution")
    report = classification_report(
        y,
        pred,
        labels=list(range(5)),
        target_names=CLASSES,
        output_dict=True,
        zero_division=0,
    )
    matrix = confusion_matrix(y, pred, labels=list(range(5)))
    robustness = {}
    for name, transformed in [
        ("rotate_90", np.rot90(x, axes=(1, 2))),
        ("horizontal_flip", x[:, :, ::-1]),
        ("brightness_0.8", np.clip(x.astype(float) * 0.8, 0, 255).astype("uint8")),
    ]:
        robustness[name] = float(accuracy_score(y, predict(transformed).argmax(1)))
    metrics = {
        "test_samples": len(y),
        "accuracy": float(accuracy_score(y, pred)),
        "macro_precision": report["macro avg"]["precision"],
        "macro_recall": report["macro avg"]["recall"],
        "macro_f1": report["macro avg"]["f1-score"],
        "per_class": report,
        "confusion_matrix": matrix.tolist(),
        "robustness_accuracy": robustness,
        "exact_pixel_overlap": False,
        "spatial_independence_verified": False,
    }
    (out / "metrics.json").write_text(json.dumps(metrics, indent=2))
    fig, ax = plt.subplots(figsize=(7, 6))
    ax.imshow(matrix, cmap="Blues")
    ax.set(
        xticks=range(5),
        yticks=range(5),
        xticklabels=CLASSES,
        yticklabels=CLASSES,
        xlabel="Predicted",
        ylabel="True",
        title="CELYRIX · held-out EuroSAT test set",
    )
    plt.setp(ax.get_xticklabels(), rotation=30, ha="right")
    for i in range(5):
        for j in range(5):
            ax.text(
                j,
                i,
                str(matrix[i, j]),
                ha="center",
                va="center",
                color="white" if matrix[i, j] > matrix.max() / 2 else "black",
            )
    fig.tight_layout()
    fig.savefig(out / "confusion_matrix.png", dpi=160)
    plt.close(fig)
    hist = json.loads((out / "history.json").read_text())
    fig, axs = plt.subplots(1, 2, figsize=(10, 4))
    for ax, key in zip(axs, ("loss", "accuracy")):
        for prefix in ("", "val_"):
            ax.plot(
                range(1, len(hist[key]) + 1), hist[prefix + key], label=prefix + key
            )
        ax.set(xlabel="Epoch", ylabel=key)
        ax.legend()
    fig.tight_layout()
    fig.savefig(out / "training_history.png", dpi=160)
    plt.close(fig)
    failures = []
    for i in np.flatnonzero(pred != y)[:30]:
        failures.append(
            {
                "filename": provenance["samples"]["test"][i]["filename"],
                "true": CLASSES[y[i]],
                "predicted": CLASSES[pred[i]],
                "max_softmax": float(probs[i].max()),
            }
        )
    (out / "failure_cases.json").write_text(json.dumps(failures, indent=2))
    np.savez_compressed(out / "test_predictions.npz", labels=y, probabilities=probs)
    print(json.dumps({k: v for k, v in metrics.items() if k != "per_class"}, indent=2))


if __name__ == "__main__":
    main()