File size: 6,085 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/env python3
"""Compare frozen-feature classifier checkpoints on identical locked manifests."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
from pathlib import Path

import joblib
import numpy as np
from scipy.stats import binomtest
from sklearn.metrics import accuracy_score, f1_score

from train_probe import apply_thresholds, metrics, predict


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--features", type=Path, required=True)
    parser.add_argument("--before", type=Path, required=True)
    parser.add_argument("--after", type=Path, required=True)
    parser.add_argument("--holdout", type=Path, required=True)
    parser.add_argument("--historical-holdout", type=Path)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()

    bundle = np.load(args.features)
    feature_metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8"))
    names = bundle["file_names"]
    classes = [str(value) for value in bundle["classes"]]
    x = bundle["features"]
    y = bundle["labels"]
    name_to_index = {str(name): index for index, name in enumerate(names)}
    checkpoints = {"before": joblib.load(args.before), "after": joblib.load(args.after)}
    for label, artifact in checkpoints.items():
        if [str(value) for value in artifact["classes"]] != classes:
            raise ValueError(f"{label} class order mismatch")
        if artifact["feature_type"] != feature_metadata["feature_type"]:
            raise ValueError(f"{label} feature schema mismatch")
        if artifact["feature_model_file_sha256"] != feature_metadata["model_file_sha256"]:
            raise ValueError(f"{label} backbone digest mismatch")
        if int(artifact["feature_dimension"]) != x.shape[1]:
            raise ValueError(f"{label} feature dimension mismatch")

    def compare(manifest: Path) -> dict[str, object]:
        with manifest.open(newline="", encoding="utf-8-sig") as handle:
            rows = list(csv.DictReader(handle))
        indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
        outputs: dict[str, object] = {}
        exact: dict[str, np.ndarray] = {}
        predictions_by_checkpoint: dict[str, np.ndarray] = {}
        for label, artifact in checkpoints.items():
            probabilities = predict(artifact, x[indices])
            outputs[label] = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"]))
            predictions = apply_thresholds(probabilities, np.asarray(artifact["thresholds"]))
            predictions_by_checkpoint[label] = predictions
            exact[label] = np.all(predictions == y[indices], axis=1)
        outputs["delta_after_minus_before"] = {
            key: outputs["after"][key] - outputs["before"][key]
            for key in ("exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc")
        }
        discordant = np.logical_xor(exact["before"], exact["after"])
        improved = int(np.sum(~exact["before"] & exact["after"]))
        regressed = int(np.sum(exact["before"] & ~exact["after"]))
        outputs["paired_exact_match"] = {
            "improved": improved,
            "regressed": regressed,
            "unchanged": int(np.sum(~discordant)),
            "exact_mcnemar_p_two_sided": (
                float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue)
                if improved + regressed else 1.0
            ),
        }
        rng = np.random.default_rng(800001)
        bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")}
        targets = y[indices]
        for _ in range(10000):
            sample = rng.integers(0, len(indices), size=len(indices))
            before_sample = predictions_by_checkpoint["before"][sample]
            after_sample = predictions_by_checkpoint["after"][sample]
            target_sample = targets[sample]
            bootstrap["exact_match_accuracy"].append(
                accuracy_score(target_sample, after_sample) - accuracy_score(target_sample, before_sample)
            )
            for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")):
                bootstrap[key].append(
                    f1_score(target_sample, after_sample, average=average, zero_division=0)
                    - f1_score(target_sample, before_sample, average=average, zero_division=0)
                )
        outputs["paired_bootstrap_delta_95pct"] = {
            key: {
                "low": float(np.quantile(values, 0.025)),
                "high": float(np.quantile(values, 0.975)),
                "resamples": 10000,
                "seed": 800001,
            }
            for key, values in bootstrap.items()
        }
        outputs["samples"] = len(indices)
        outputs["file_names"] = [row["file_name"] for row in rows]
        return outputs

    report = {
        "protocol": "Paired before/after evaluation on identical locked data; frozen backbone and feature schema verified by SHA-256.",
        "features": str(args.features.resolve()),
        "features_sha256": sha256(args.features),
        "checkpoints": {
            "before": {"path": str(args.before.resolve()), "sha256": sha256(args.before)},
            "after": {"path": str(args.after.resolve()), "sha256": sha256(args.after)},
        },
        "new_holdout32": compare(args.holdout),
    }
    if args.historical_holdout:
        report["historical_holdout32"] = compare(args.historical_holdout)
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()