#!/usr/bin/env python3 """Cross-fitted confidence gates between complementary v801 experts.""" from __future__ import annotations import argparse import csv import json from pathlib import Path import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from scipy.special import logsumexp from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score from sklearn.preprocessing import StandardScaler from train_v801 import ( EPS, Spec, all_labelsets, choose_thresholds, labelset_ids, make_variant, meta_features, oof_probabilities, report_metrics, tune_structured_decoder, ) def expert_outputs( y: np.ndarray, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], origins: np.ndarray, ) -> dict[str, object]: decoder, tuned_predictions, metrics = tune_structured_decoder( y, probabilities, folds, origins ) labelsets = all_labelsets(y.shape[1]) set_ids = labelset_ids(y, labelsets) cardinalities = labelsets.sum(axis=1).astype(int) - 1 predictions = np.zeros_like(y) confidence = { name: np.zeros(len(y), dtype=np.float64) for name in ( "joint_margin", "joint_top_probability", "bernoulli_set_score", "negative_entropy", "mean_binary_margin", "min_binary_margin", "cardinality_confidence", ) } p = np.clip(probabilities, EPS, 1 - EPS) binary_entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p)) for train, valid in folds: thresholds = np.clip(choose_thresholds(y[train], p[train]), EPS, 1 - EPS) logits = np.log(p[valid] / (1 - p[valid])) logits -= float(decoder["threshold_bias_scale"]) * np.log( thresholds / (1 - thresholds) ) counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) smoothing = float(decoder["prior_smoothing"]) log_prior = np.log( (counts + smoothing) / (len(train) + smoothing * len(labelsets)) ) cardinality = LogisticRegression( C=1.0, max_iter=2000, class_weight="balanced", random_state=801 ).fit(meta_features(p[train]), y[train].sum(axis=1)) card_probability = np.clip( cardinality.predict_proba(meta_features(p[valid])), EPS, 1 ) score = logits @ labelsets.T score += float(decoder["labelset_prior_weight"]) * log_prior score += float(decoder["cardinality_weight"]) * np.log( card_probability[:, cardinalities] ) order = np.argsort(score, axis=1) best = order[:, -1] predictions[valid] = labelsets[best] confidence["joint_margin"][valid] = score[np.arange(len(valid)), order[:, -1]] - score[ np.arange(len(valid)), order[:, -2] ] confidence["joint_top_probability"][valid] = np.exp( score[np.arange(len(valid)), best] - logsumexp(score, axis=1) ) selected = labelsets[best] confidence["bernoulli_set_score"][valid] = ( selected * np.log(p[valid]) + (1 - selected) * np.log(1 - p[valid]) ).sum(axis=1) confidence["negative_entropy"][valid] = -binary_entropy[valid].sum(axis=1) margin = np.abs(p[valid] - thresholds) confidence["mean_binary_margin"][valid] = margin.mean(axis=1) confidence["min_binary_margin"][valid] = margin.min(axis=1) confidence["cardinality_confidence"][valid] = card_probability.max(axis=1) if not np.array_equal(predictions, tuned_predictions): raise RuntimeError("expert score reconstruction does not match tuned predictions") return { "decoder": decoder, "predictions": predictions, "probabilities": probabilities, "metrics": metrics, "confidence": confidence, } def prediction_metrics( y: np.ndarray, probabilities: np.ndarray, predictions: np.ndarray, recent: np.ndarray, ) -> dict[str, object]: metrics = report_metrics(y, probabilities, predictions) metrics["incremental100_exact_match_accuracy"] = float( accuracy_score(y[recent], predictions[recent]) ) return metrics def threshold_gate( y: np.ndarray, legacy: dict[str, object], patch: dict[str, object], folds: list[tuple[np.ndarray, np.ndarray]], recent: np.ndarray, metric: str, ) -> tuple[np.ndarray, np.ndarray, list[float], dict[str, object]]: delta = patch["confidence"][metric] - legacy["confidence"][metric] legacy_correct = np.all(legacy["predictions"] == y, axis=1) patch_correct = np.all(patch["predictions"] == y, axis=1) use_patch = np.zeros(len(y), dtype=bool) thresholds = [] for train, valid in folds: values = np.unique(delta[train]) candidates = np.concatenate( ((-np.inf,), (values[:-1] + values[1:]) / 2, (np.inf,)) ) best = None for threshold in candidates: choose_patch = delta[train] >= threshold correct = np.where(choose_patch, patch_correct[train], legacy_correct[train]) key = ( float(correct.mean()), float(correct[recent[train]].mean()) if np.any(recent[train]) else 0.0, -abs(float(threshold)) if np.isfinite(threshold) else -1e12, ) if best is None or key > best[0]: best = key, float(threshold) assert best is not None thresholds.append(best[1]) use_patch[valid] = delta[valid] >= best[1] predictions = np.where( use_patch[:, None], patch["predictions"], legacy["predictions"] ) probabilities = np.where( use_patch[:, None], patch["probabilities"], legacy["probabilities"] ) return predictions, probabilities, thresholds, prediction_metrics( y, probabilities, predictions, recent ) def logistic_gate( y: np.ndarray, legacy: dict[str, object], patch: dict[str, object], folds: list[tuple[np.ndarray, np.ndarray]], recent: np.ndarray, c: float, ) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: confidence_names = sorted(legacy["confidence"]) confidence_delta = np.column_stack( [patch["confidence"][name] - legacy["confidence"][name] for name in confidence_names] ) x = np.column_stack( ( confidence_delta, patch["probabilities"] - legacy["probabilities"], patch["predictions"].astype(np.float64) - legacy["predictions"].astype(np.float64), patch["predictions"].sum(axis=1).astype(np.float64) - legacy["predictions"].sum(axis=1).astype(np.float64), ) ) legacy_correct = np.all(legacy["predictions"] == y, axis=1) patch_correct = np.all(patch["predictions"] == y, axis=1) discordant = legacy_correct != patch_correct target = patch_correct.astype(np.uint8) use_patch = np.ones(len(y), dtype=bool) for train, valid in folds: gate_train = train[discordant[train]] scaler = StandardScaler().fit(x[gate_train]) model = LogisticRegression( C=c, solver="liblinear", class_weight="balanced", random_state=801, ).fit(scaler.transform(x[gate_train]), target[gate_train]) use_patch[valid] = model.predict_proba(scaler.transform(x[valid]))[:, 1] >= 0.5 predictions = np.where( use_patch[:, None], patch["predictions"], legacy["predictions"] ) probabilities = np.where( use_patch[:, None], patch["probabilities"], legacy["probabilities"] ) return predictions, probabilities, prediction_metrics( y, probabilities, predictions, recent ) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--development", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--seed", type=int, default=801208) args = parser.parse_args() bundle = np.load(args.features) names = bundle["file_names"].astype(str) y_all = bundle["labels"].astype(np.uint8) lookup = {name: index for index, name in enumerate(names)} with args.development.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) development_idx = np.asarray([lookup[row["file_name"]] for row in rows]) origins = np.full(len(names), "excluded", dtype="U32") for row in rows: origins[lookup[row["file_name"]]] = row["origin"] anchor_idx = development_idx[origins[development_idx] == "old440"] target_idx = development_idx[origins[development_idx] != "old440"] y = y_all[target_idx] target_origins = origins[target_idx] recent = target_origins == "incremental100" folds = list( MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( np.zeros(len(target_idx)), y ) ) specs = { "legacy": Spec("legacy", 0.003, class_weight=None), "patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None), } experts = {} for name, spec in specs.items(): probabilities = oof_probabilities( make_variant(bundle, spec.variant), y_all, origins, anchor_idx, target_idx, folds, spec, ) experts[name] = expert_outputs(y, probabilities, folds, target_origins) print( f"{name}: {experts[name]['metrics']['exact_match_accuracy']:.4f}", flush=True, ) legacy_correct = np.all(experts["legacy"]["predictions"] == y, axis=1) patch_correct = np.all(experts["patch"]["predictions"] == y, axis=1) reports = [] for metric in sorted(experts["legacy"]["confidence"]): predictions, probabilities, thresholds, metrics = threshold_gate( y, experts["legacy"], experts["patch"], folds, recent, metric ) reports.append( { "name": f"threshold-{metric}", "fold_thresholds": thresholds, "metrics": metrics, } ) print( f"threshold-{metric}: exact={metrics['exact_match_accuracy']:.4f} " f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True, ) entropy_delta = ( experts["patch"]["confidence"]["negative_entropy"] - experts["legacy"]["confidence"]["negative_entropy"] ) for threshold in (-0.1, -0.05, 0.0, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2): use_patch = entropy_delta >= threshold predictions = np.where( use_patch[:, None], experts["patch"]["predictions"], experts["legacy"]["predictions"], ) probabilities = np.where( use_patch[:, None], experts["patch"]["probabilities"], experts["legacy"]["probabilities"], ) metrics = prediction_metrics(y, probabilities, predictions, recent) reports.append( { "name": f"fixed-negative-entropy-{threshold:g}", "threshold": threshold, "patch_fraction": float(use_patch.mean()), "metrics": metrics, } ) print( f"fixed-negative-entropy-{threshold:g}: " f"exact={metrics['exact_match_accuracy']:.4f} " f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True, ) for c in (0.001, 0.01, 0.1, 1.0, 10.0): _, _, metrics = logistic_gate( y, experts["legacy"], experts["patch"], folds, recent, c ) reports.append({"name": f"logistic-C{c:g}", "metrics": metrics}) print( f"logistic-C{c:g}: exact={metrics['exact_match_accuracy']:.4f} " f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True, ) reports.sort( key=lambda report: ( float(report["metrics"]["exact_match_accuracy"]), float(report["metrics"]["incremental100_exact_match_accuracy"]), float(report["metrics"]["macro_f1"]), ), reverse=True, ) output = { "protocol": "development736 only; every gate prediction is cross-fitted", "seed": args.seed, "complementarity": { "legacy_correct": int(legacy_correct.sum()), "patch_correct": int(patch_correct.sum()), "both_correct": int(np.sum(legacy_correct & patch_correct)), "legacy_only": int(np.sum(legacy_correct & ~patch_correct)), "patch_only": int(np.sum(~legacy_correct & patch_correct)), "neither": int(np.sum(~legacy_correct & ~patch_correct)), "oracle_exact_match_accuracy": float(np.mean(legacy_correct | patch_correct)), }, "experts": { name: {"decoder": expert["decoder"], "metrics": expert["metrics"]} for name, expert in experts.items() }, "gate_candidates": reports, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") print(json.dumps(reports[0], indent=2), flush=True) if __name__ == "__main__": main()