#!/usr/bin/env python3 """Historical-reference check for fixed v801 local decoder candidates.""" from __future__ import annotations import argparse import csv import json from pathlib import Path import joblib import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from sklearn.linear_model import LogisticRegression from experiment_v801_knn import knn_oof from train_probe import apply_thresholds, predict as legacy_predict from train_v801 import ( Spec, all_labelsets, artifact_probabilities, binary_probabilities, choose_thresholds, fit_binary, labelset_ids, make_variant, meta_features, oof_probabilities, report_metrics, structured_predictions, threshold_predictions, ) def knn_probabilities( query: np.ndarray, references: np.ndarray, labels: np.ndarray, k: int = 11, temperature: float = 0.1, ) -> np.ndarray: similarity = query @ references.T neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:] neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1) neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True) weights = np.exp(neighbour_similarity / temperature) weights /= weights.sum(axis=1, keepdims=True) return np.einsum("nk,nkc->nc", weights, labels[neighbours]) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--legacy-features", type=Path, required=True) parser.add_argument("--development", type=Path, required=True) parser.add_argument("--historical", type=Path, required=True) parser.add_argument("--v800", type=Path, required=True) parser.add_argument("--stage1-v801", type=Path, required=True) parser.add_argument("--robust-pattern", required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--seed", type=int, default=801208) args = parser.parse_args() bundle = np.load(args.features) legacy_bundle = np.load(args.legacy_features) names = bundle["file_names"].astype(str) y = bundle["labels"].astype(np.uint8) lookup = {name: index for index, name in enumerate(names)} with args.development.open(newline="", encoding="utf-8-sig") as handle: development_rows = list(csv.DictReader(handle)) with args.historical.open(newline="", encoding="utf-8-sig") as handle: historical_rows = list(csv.DictReader(handle)) development_idx = np.asarray([lookup[row["file_name"]] for row in development_rows]) historical_idx = np.asarray([lookup[row["file_name"]] for row in historical_rows]) origins = np.full(len(names), "excluded", dtype="U32") for row in development_rows: origins[lookup[row["file_name"]]] = row["origin"] anchor_idx = development_idx[origins[development_idx] == "old440"] target_idx = development_idx[origins[development_idx] != "old440"] folds = list( MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( np.zeros(len(target_idx)), y[target_idx] ) ) legacy_x = make_variant(bundle, "legacy") patch_x = make_variant(bundle, "meanmax_clspatchmax") spec = Spec("legacy", 0.003, class_weight=None) legacy_oof = oof_probabilities( legacy_x, y, origins, anchor_idx, target_idx, folds, spec ) neighbour_oof = knn_oof( patch_x, y, anchor_idx, target_idx, folds, k=11, temperature=0.1 ) mixed_oof = 0.8 * legacy_oof + 0.2 * neighbour_oof thresholds = choose_thresholds(y[target_idx], mixed_oof) labelsets = all_labelsets(y.shape[1]) set_ids = labelset_ids(y[target_idx], labelsets) counts = np.bincount(set_ids, minlength=len(labelsets)).astype(np.float64) labelset_log_prior = np.log((counts + 1.0) / (len(target_idx) + len(labelsets))) cardinality_model = LogisticRegression( C=1.0, max_iter=2000, class_weight="balanced", random_state=801 ).fit(meta_features(mixed_oof), y[target_idx].sum(axis=1)) probe = fit_binary( legacy_x[development_idx], y[development_idx], spec, np.ones(len(development_idx), dtype=np.float64), ) legacy_probability = binary_probabilities(probe, legacy_x[historical_idx]) neighbour_probability = knn_probabilities( patch_x[historical_idx], patch_x[development_idx], y[development_idx] ) mixed_probability = 0.8 * legacy_probability + 0.2 * neighbour_probability targets = y[historical_idx] decoder_candidates = { "q0-prior0.5-card0.25": (0.0, 0.5, 0.25), "q0.25-prior0.5-card0.5": (0.25, 0.5, 0.5), "q0.25-prior0.75-card1": (0.25, 0.75, 1.0), "q0.5-prior0.75-card1": (0.5, 0.75, 1.0), "q0.5-prior0.75-card1.5": (0.5, 0.75, 1.5), } robust_paths = sorted(Path().glob(args.robust_pattern)) robust = {} for name in decoder_candidates: values = [] recent_values = [] for path in robust_paths: report = json.loads(path.read_text()) metrics = report["fixed_decoders"][name]["metrics"] values.append(metrics["exact_match_accuracy"]) recent_values.append(metrics["incremental100_exact_match_accuracy"]) robust[name] = { "seeds": len(values), "mean_exact_match_accuracy": float(np.mean(values)), "min_exact_match_accuracy": float(np.min(values)), "mean_incremental100_exact_match_accuracy": float(np.mean(recent_values)), } candidates = {} for name, (q, prior, cardinality) in decoder_candidates.items(): artifact = { "thresholds": thresholds, "decoder": { "threshold_bias_scale": q, "labelset_prior_weight": prior, "cardinality_weight": cardinality, "prior_smoothing": 1.0, }, "labelsets": labelsets, "labelset_log_prior": labelset_log_prior, "cardinality_model": cardinality_model, } predictions = structured_predictions(artifact, mixed_probability) candidates[name] = { "decoder": artifact["decoder"], "robust_development": robust[name], "historical_metrics": report_metrics(targets, mixed_probability, predictions), } threshold_prediction = threshold_predictions(mixed_probability, thresholds) v800 = joblib.load(args.v800) v800_probability = legacy_predict(v800, legacy_bundle["features"][historical_idx]) v800_prediction = apply_thresholds(v800_probability, np.asarray(v800["thresholds"])) stage1 = joblib.load(args.stage1_v801) stage1_probability = artifact_probabilities( stage1, bundle["view_descriptors"][historical_idx], bundle["features"][historical_idx] ) stage1_prediction = structured_predictions(stage1, stage1_probability) output = { "protocol": "historical32 is a non-independent reference; new_holdout32 is not read", "robust_files": [str(path) for path in robust_paths], "baselines": { "v800": report_metrics(targets, v800_probability, v800_prediction), "stage1_v801": report_metrics(targets, stage1_probability, stage1_prediction), }, "local_candidates": candidates, "local_crossfit_threshold_historical": report_metrics( targets, mixed_probability, threshold_prediction ), } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") print(json.dumps(output, indent=2), flush=True) if __name__ == "__main__": main()