File size: 7,746 Bytes
59839e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | #!/usr/bin/env python3
"""Historical-reference check for fixed v801 local decoder candidates."""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import joblib
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from experiment_v801_knn import knn_oof
from train_probe import apply_thresholds, predict as legacy_predict
from train_v801 import (
Spec,
all_labelsets,
artifact_probabilities,
binary_probabilities,
choose_thresholds,
fit_binary,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
structured_predictions,
threshold_predictions,
)
def knn_probabilities(
query: np.ndarray,
references: np.ndarray,
labels: np.ndarray,
k: int = 11,
temperature: float = 0.1,
) -> np.ndarray:
similarity = query @ references.T
neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:]
neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1)
neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True)
weights = np.exp(neighbour_similarity / temperature)
weights /= weights.sum(axis=1, keepdims=True)
return np.einsum("nk,nkc->nc", weights, labels[neighbours])
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--legacy-features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--historical", type=Path, required=True)
parser.add_argument("--v800", type=Path, required=True)
parser.add_argument("--stage1-v801", type=Path, required=True)
parser.add_argument("--robust-pattern", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
bundle = np.load(args.features)
legacy_bundle = np.load(args.legacy_features)
names = bundle["file_names"].astype(str)
y = bundle["labels"].astype(np.uint8)
lookup = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
development_rows = list(csv.DictReader(handle))
with args.historical.open(newline="", encoding="utf-8-sig") as handle:
historical_rows = list(csv.DictReader(handle))
development_idx = np.asarray([lookup[row["file_name"]] for row in development_rows])
historical_idx = np.asarray([lookup[row["file_name"]] for row in historical_rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in development_rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
folds = list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split(
np.zeros(len(target_idx)), y[target_idx]
)
)
legacy_x = make_variant(bundle, "legacy")
patch_x = make_variant(bundle, "meanmax_clspatchmax")
spec = Spec("legacy", 0.003, class_weight=None)
legacy_oof = oof_probabilities(
legacy_x, y, origins, anchor_idx, target_idx, folds, spec
)
neighbour_oof = knn_oof(
patch_x, y, anchor_idx, target_idx, folds, k=11, temperature=0.1
)
mixed_oof = 0.8 * legacy_oof + 0.2 * neighbour_oof
thresholds = choose_thresholds(y[target_idx], mixed_oof)
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y[target_idx], labelsets)
counts = np.bincount(set_ids, minlength=len(labelsets)).astype(np.float64)
labelset_log_prior = np.log((counts + 1.0) / (len(target_idx) + len(labelsets)))
cardinality_model = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(meta_features(mixed_oof), y[target_idx].sum(axis=1))
probe = fit_binary(
legacy_x[development_idx], y[development_idx], spec,
np.ones(len(development_idx), dtype=np.float64),
)
legacy_probability = binary_probabilities(probe, legacy_x[historical_idx])
neighbour_probability = knn_probabilities(
patch_x[historical_idx], patch_x[development_idx], y[development_idx]
)
mixed_probability = 0.8 * legacy_probability + 0.2 * neighbour_probability
targets = y[historical_idx]
decoder_candidates = {
"q0-prior0.5-card0.25": (0.0, 0.5, 0.25),
"q0.25-prior0.5-card0.5": (0.25, 0.5, 0.5),
"q0.25-prior0.75-card1": (0.25, 0.75, 1.0),
"q0.5-prior0.75-card1": (0.5, 0.75, 1.0),
"q0.5-prior0.75-card1.5": (0.5, 0.75, 1.5),
}
robust_paths = sorted(Path().glob(args.robust_pattern))
robust = {}
for name in decoder_candidates:
values = []
recent_values = []
for path in robust_paths:
report = json.loads(path.read_text())
metrics = report["fixed_decoders"][name]["metrics"]
values.append(metrics["exact_match_accuracy"])
recent_values.append(metrics["incremental100_exact_match_accuracy"])
robust[name] = {
"seeds": len(values),
"mean_exact_match_accuracy": float(np.mean(values)),
"min_exact_match_accuracy": float(np.min(values)),
"mean_incremental100_exact_match_accuracy": float(np.mean(recent_values)),
}
candidates = {}
for name, (q, prior, cardinality) in decoder_candidates.items():
artifact = {
"thresholds": thresholds,
"decoder": {
"threshold_bias_scale": q,
"labelset_prior_weight": prior,
"cardinality_weight": cardinality,
"prior_smoothing": 1.0,
},
"labelsets": labelsets,
"labelset_log_prior": labelset_log_prior,
"cardinality_model": cardinality_model,
}
predictions = structured_predictions(artifact, mixed_probability)
candidates[name] = {
"decoder": artifact["decoder"],
"robust_development": robust[name],
"historical_metrics": report_metrics(targets, mixed_probability, predictions),
}
threshold_prediction = threshold_predictions(mixed_probability, thresholds)
v800 = joblib.load(args.v800)
v800_probability = legacy_predict(v800, legacy_bundle["features"][historical_idx])
v800_prediction = apply_thresholds(v800_probability, np.asarray(v800["thresholds"]))
stage1 = joblib.load(args.stage1_v801)
stage1_probability = artifact_probabilities(
stage1, bundle["view_descriptors"][historical_idx], bundle["features"][historical_idx]
)
stage1_prediction = structured_predictions(stage1, stage1_probability)
output = {
"protocol": "historical32 is a non-independent reference; new_holdout32 is not read",
"robust_files": [str(path) for path in robust_paths],
"baselines": {
"v800": report_metrics(targets, v800_probability, v800_prediction),
"stage1_v801": report_metrics(targets, stage1_probability, stage1_prediction),
},
"local_candidates": candidates,
"local_crossfit_threshold_historical": report_metrics(
targets, mixed_probability, threshold_prediction
),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
print(json.dumps(output, indent=2), flush=True)
if __name__ == "__main__":
main()
|