v802-classifier / src /experiment_v801_historical_reference.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
7.75 kB
#!/usr/bin/env python3
"""Historical-reference check for fixed v801 local decoder candidates."""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import joblib
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from experiment_v801_knn import knn_oof
from train_probe import apply_thresholds, predict as legacy_predict
from train_v801 import (
Spec,
all_labelsets,
artifact_probabilities,
binary_probabilities,
choose_thresholds,
fit_binary,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
structured_predictions,
threshold_predictions,
)
def knn_probabilities(
query: np.ndarray,
references: np.ndarray,
labels: np.ndarray,
k: int = 11,
temperature: float = 0.1,
) -> np.ndarray:
similarity = query @ references.T
neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:]
neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1)
neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True)
weights = np.exp(neighbour_similarity / temperature)
weights /= weights.sum(axis=1, keepdims=True)
return np.einsum("nk,nkc->nc", weights, labels[neighbours])
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--legacy-features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--historical", type=Path, required=True)
parser.add_argument("--v800", type=Path, required=True)
parser.add_argument("--stage1-v801", type=Path, required=True)
parser.add_argument("--robust-pattern", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
bundle = np.load(args.features)
legacy_bundle = np.load(args.legacy_features)
names = bundle["file_names"].astype(str)
y = bundle["labels"].astype(np.uint8)
lookup = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
development_rows = list(csv.DictReader(handle))
with args.historical.open(newline="", encoding="utf-8-sig") as handle:
historical_rows = list(csv.DictReader(handle))
development_idx = np.asarray([lookup[row["file_name"]] for row in development_rows])
historical_idx = np.asarray([lookup[row["file_name"]] for row in historical_rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in development_rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
folds = list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split(
np.zeros(len(target_idx)), y[target_idx]
)
)
legacy_x = make_variant(bundle, "legacy")
patch_x = make_variant(bundle, "meanmax_clspatchmax")
spec = Spec("legacy", 0.003, class_weight=None)
legacy_oof = oof_probabilities(
legacy_x, y, origins, anchor_idx, target_idx, folds, spec
)
neighbour_oof = knn_oof(
patch_x, y, anchor_idx, target_idx, folds, k=11, temperature=0.1
)
mixed_oof = 0.8 * legacy_oof + 0.2 * neighbour_oof
thresholds = choose_thresholds(y[target_idx], mixed_oof)
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y[target_idx], labelsets)
counts = np.bincount(set_ids, minlength=len(labelsets)).astype(np.float64)
labelset_log_prior = np.log((counts + 1.0) / (len(target_idx) + len(labelsets)))
cardinality_model = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(meta_features(mixed_oof), y[target_idx].sum(axis=1))
probe = fit_binary(
legacy_x[development_idx], y[development_idx], spec,
np.ones(len(development_idx), dtype=np.float64),
)
legacy_probability = binary_probabilities(probe, legacy_x[historical_idx])
neighbour_probability = knn_probabilities(
patch_x[historical_idx], patch_x[development_idx], y[development_idx]
)
mixed_probability = 0.8 * legacy_probability + 0.2 * neighbour_probability
targets = y[historical_idx]
decoder_candidates = {
"q0-prior0.5-card0.25": (0.0, 0.5, 0.25),
"q0.25-prior0.5-card0.5": (0.25, 0.5, 0.5),
"q0.25-prior0.75-card1": (0.25, 0.75, 1.0),
"q0.5-prior0.75-card1": (0.5, 0.75, 1.0),
"q0.5-prior0.75-card1.5": (0.5, 0.75, 1.5),
}
robust_paths = sorted(Path().glob(args.robust_pattern))
robust = {}
for name in decoder_candidates:
values = []
recent_values = []
for path in robust_paths:
report = json.loads(path.read_text())
metrics = report["fixed_decoders"][name]["metrics"]
values.append(metrics["exact_match_accuracy"])
recent_values.append(metrics["incremental100_exact_match_accuracy"])
robust[name] = {
"seeds": len(values),
"mean_exact_match_accuracy": float(np.mean(values)),
"min_exact_match_accuracy": float(np.min(values)),
"mean_incremental100_exact_match_accuracy": float(np.mean(recent_values)),
}
candidates = {}
for name, (q, prior, cardinality) in decoder_candidates.items():
artifact = {
"thresholds": thresholds,
"decoder": {
"threshold_bias_scale": q,
"labelset_prior_weight": prior,
"cardinality_weight": cardinality,
"prior_smoothing": 1.0,
},
"labelsets": labelsets,
"labelset_log_prior": labelset_log_prior,
"cardinality_model": cardinality_model,
}
predictions = structured_predictions(artifact, mixed_probability)
candidates[name] = {
"decoder": artifact["decoder"],
"robust_development": robust[name],
"historical_metrics": report_metrics(targets, mixed_probability, predictions),
}
threshold_prediction = threshold_predictions(mixed_probability, thresholds)
v800 = joblib.load(args.v800)
v800_probability = legacy_predict(v800, legacy_bundle["features"][historical_idx])
v800_prediction = apply_thresholds(v800_probability, np.asarray(v800["thresholds"]))
stage1 = joblib.load(args.stage1_v801)
stage1_probability = artifact_probabilities(
stage1, bundle["view_descriptors"][historical_idx], bundle["features"][historical_idx]
)
stage1_prediction = structured_predictions(stage1, stage1_probability)
output = {
"protocol": "historical32 is a non-independent reference; new_holdout32 is not read",
"robust_files": [str(path) for path in robust_paths],
"baselines": {
"v800": report_metrics(targets, v800_probability, v800_prediction),
"stage1_v801": report_metrics(targets, stage1_probability, stage1_prediction),
},
"local_candidates": candidates,
"local_crossfit_threshold_historical": report_metrics(
targets, mixed_probability, threshold_prediction
),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
print(json.dumps(output, indent=2), flush=True)
if __name__ == "__main__":
main()