Download src/experiment_v801_gate.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 13.5 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_gate.py
- Command line
-
hf download hf://sneo/v802-classifier/src/experiment_v801_gate.py
-
curl -L -o experiment_v801_gate.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/experiment_v801_gate.py
13.5 kB
| #!/usr/bin/env python3 | |
| """Cross-fitted confidence gates between complementary v801 experts.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedKFold | |
| from scipy.special import logsumexp | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.metrics import accuracy_score | |
| from sklearn.preprocessing import StandardScaler | |
| from train_v801 import ( | |
| EPS, | |
| Spec, | |
| all_labelsets, | |
| choose_thresholds, | |
| labelset_ids, | |
| make_variant, | |
| meta_features, | |
| oof_probabilities, | |
| report_metrics, | |
| tune_structured_decoder, | |
| ) | |
| def expert_outputs( | |
| y: np.ndarray, | |
| probabilities: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| origins: np.ndarray, | |
| ) -> dict[str, object]: | |
| decoder, tuned_predictions, metrics = tune_structured_decoder( | |
| y, probabilities, folds, origins | |
| ) | |
| labelsets = all_labelsets(y.shape[1]) | |
| set_ids = labelset_ids(y, labelsets) | |
| cardinalities = labelsets.sum(axis=1).astype(int) - 1 | |
| predictions = np.zeros_like(y) | |
| confidence = { | |
| name: np.zeros(len(y), dtype=np.float64) | |
| for name in ( | |
| "joint_margin", | |
| "joint_top_probability", | |
| "bernoulli_set_score", | |
| "negative_entropy", | |
| "mean_binary_margin", | |
| "min_binary_margin", | |
| "cardinality_confidence", | |
| ) | |
| } | |
| p = np.clip(probabilities, EPS, 1 - EPS) | |
| binary_entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p)) | |
| for train, valid in folds: | |
| thresholds = np.clip(choose_thresholds(y[train], p[train]), EPS, 1 - EPS) | |
| logits = np.log(p[valid] / (1 - p[valid])) | |
| logits -= float(decoder["threshold_bias_scale"]) * np.log( | |
| thresholds / (1 - thresholds) | |
| ) | |
| counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) | |
| smoothing = float(decoder["prior_smoothing"]) | |
| log_prior = np.log( | |
| (counts + smoothing) / (len(train) + smoothing * len(labelsets)) | |
| ) | |
| cardinality = LogisticRegression( | |
| C=1.0, max_iter=2000, class_weight="balanced", random_state=801 | |
| ).fit(meta_features(p[train]), y[train].sum(axis=1)) | |
| card_probability = np.clip( | |
| cardinality.predict_proba(meta_features(p[valid])), EPS, 1 | |
| ) | |
| score = logits @ labelsets.T | |
| score += float(decoder["labelset_prior_weight"]) * log_prior | |
| score += float(decoder["cardinality_weight"]) * np.log( | |
| card_probability[:, cardinalities] | |
| ) | |
| order = np.argsort(score, axis=1) | |
| best = order[:, -1] | |
| predictions[valid] = labelsets[best] | |
| confidence["joint_margin"][valid] = score[np.arange(len(valid)), order[:, -1]] - score[ | |
| np.arange(len(valid)), order[:, -2] | |
| ] | |
| confidence["joint_top_probability"][valid] = np.exp( | |
| score[np.arange(len(valid)), best] - logsumexp(score, axis=1) | |
| ) | |
| selected = labelsets[best] | |
| confidence["bernoulli_set_score"][valid] = ( | |
| selected * np.log(p[valid]) + (1 - selected) * np.log(1 - p[valid]) | |
| ).sum(axis=1) | |
| confidence["negative_entropy"][valid] = -binary_entropy[valid].sum(axis=1) | |
| margin = np.abs(p[valid] - thresholds) | |
| confidence["mean_binary_margin"][valid] = margin.mean(axis=1) | |
| confidence["min_binary_margin"][valid] = margin.min(axis=1) | |
| confidence["cardinality_confidence"][valid] = card_probability.max(axis=1) | |
| if not np.array_equal(predictions, tuned_predictions): | |
| raise RuntimeError("expert score reconstruction does not match tuned predictions") | |
| return { | |
| "decoder": decoder, | |
| "predictions": predictions, | |
| "probabilities": probabilities, | |
| "metrics": metrics, | |
| "confidence": confidence, | |
| } | |
| def prediction_metrics( | |
| y: np.ndarray, | |
| probabilities: np.ndarray, | |
| predictions: np.ndarray, | |
| recent: np.ndarray, | |
| ) -> dict[str, object]: | |
| metrics = report_metrics(y, probabilities, predictions) | |
| metrics["incremental100_exact_match_accuracy"] = float( | |
| accuracy_score(y[recent], predictions[recent]) | |
| ) | |
| return metrics | |
| def threshold_gate( | |
| y: np.ndarray, | |
| legacy: dict[str, object], | |
| patch: dict[str, object], | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| recent: np.ndarray, | |
| metric: str, | |
| ) -> tuple[np.ndarray, np.ndarray, list[float], dict[str, object]]: | |
| delta = patch["confidence"][metric] - legacy["confidence"][metric] | |
| legacy_correct = np.all(legacy["predictions"] == y, axis=1) | |
| patch_correct = np.all(patch["predictions"] == y, axis=1) | |
| use_patch = np.zeros(len(y), dtype=bool) | |
| thresholds = [] | |
| for train, valid in folds: | |
| values = np.unique(delta[train]) | |
| candidates = np.concatenate( | |
| ((-np.inf,), (values[:-1] + values[1:]) / 2, (np.inf,)) | |
| ) | |
| best = None | |
| for threshold in candidates: | |
| choose_patch = delta[train] >= threshold | |
| correct = np.where(choose_patch, patch_correct[train], legacy_correct[train]) | |
| key = ( | |
| float(correct.mean()), | |
| float(correct[recent[train]].mean()) if np.any(recent[train]) else 0.0, | |
| -abs(float(threshold)) if np.isfinite(threshold) else -1e12, | |
| ) | |
| if best is None or key > best[0]: | |
| best = key, float(threshold) | |
| assert best is not None | |
| thresholds.append(best[1]) | |
| use_patch[valid] = delta[valid] >= best[1] | |
| predictions = np.where( | |
| use_patch[:, None], patch["predictions"], legacy["predictions"] | |
| ) | |
| probabilities = np.where( | |
| use_patch[:, None], patch["probabilities"], legacy["probabilities"] | |
| ) | |
| return predictions, probabilities, thresholds, prediction_metrics( | |
| y, probabilities, predictions, recent | |
| ) | |
| def logistic_gate( | |
| y: np.ndarray, | |
| legacy: dict[str, object], | |
| patch: dict[str, object], | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| recent: np.ndarray, | |
| c: float, | |
| ) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: | |
| confidence_names = sorted(legacy["confidence"]) | |
| confidence_delta = np.column_stack( | |
| [patch["confidence"][name] - legacy["confidence"][name] for name in confidence_names] | |
| ) | |
| x = np.column_stack( | |
| ( | |
| confidence_delta, | |
| patch["probabilities"] - legacy["probabilities"], | |
| patch["predictions"].astype(np.float64) | |
| - legacy["predictions"].astype(np.float64), | |
| patch["predictions"].sum(axis=1).astype(np.float64) | |
| - legacy["predictions"].sum(axis=1).astype(np.float64), | |
| ) | |
| ) | |
| legacy_correct = np.all(legacy["predictions"] == y, axis=1) | |
| patch_correct = np.all(patch["predictions"] == y, axis=1) | |
| discordant = legacy_correct != patch_correct | |
| target = patch_correct.astype(np.uint8) | |
| use_patch = np.ones(len(y), dtype=bool) | |
| for train, valid in folds: | |
| gate_train = train[discordant[train]] | |
| scaler = StandardScaler().fit(x[gate_train]) | |
| model = LogisticRegression( | |
| C=c, | |
| solver="liblinear", | |
| class_weight="balanced", | |
| random_state=801, | |
| ).fit(scaler.transform(x[gate_train]), target[gate_train]) | |
| use_patch[valid] = model.predict_proba(scaler.transform(x[valid]))[:, 1] >= 0.5 | |
| predictions = np.where( | |
| use_patch[:, None], patch["predictions"], legacy["predictions"] | |
| ) | |
| probabilities = np.where( | |
| use_patch[:, None], patch["probabilities"], legacy["probabilities"] | |
| ) | |
| return predictions, probabilities, prediction_metrics( | |
| y, probabilities, predictions, recent | |
| ) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--development", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=801208) | |
| args = parser.parse_args() | |
| bundle = np.load(args.features) | |
| names = bundle["file_names"].astype(str) | |
| y_all = bundle["labels"].astype(np.uint8) | |
| lookup = {name: index for index, name in enumerate(names)} | |
| with args.development.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| development_idx = np.asarray([lookup[row["file_name"]] for row in rows]) | |
| origins = np.full(len(names), "excluded", dtype="U32") | |
| for row in rows: | |
| origins[lookup[row["file_name"]]] = row["origin"] | |
| anchor_idx = development_idx[origins[development_idx] == "old440"] | |
| target_idx = development_idx[origins[development_idx] != "old440"] | |
| y = y_all[target_idx] | |
| target_origins = origins[target_idx] | |
| recent = target_origins == "incremental100" | |
| folds = list( | |
| MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split( | |
| np.zeros(len(target_idx)), y | |
| ) | |
| ) | |
| specs = { | |
| "legacy": Spec("legacy", 0.003, class_weight=None), | |
| "patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None), | |
| } | |
| experts = {} | |
| for name, spec in specs.items(): | |
| probabilities = oof_probabilities( | |
| make_variant(bundle, spec.variant), y_all, origins, | |
| anchor_idx, target_idx, folds, spec, | |
| ) | |
| experts[name] = expert_outputs(y, probabilities, folds, target_origins) | |
| print( | |
| f"{name}: {experts[name]['metrics']['exact_match_accuracy']:.4f}", | |
| flush=True, | |
| ) | |
| legacy_correct = np.all(experts["legacy"]["predictions"] == y, axis=1) | |
| patch_correct = np.all(experts["patch"]["predictions"] == y, axis=1) | |
| reports = [] | |
| for metric in sorted(experts["legacy"]["confidence"]): | |
| predictions, probabilities, thresholds, metrics = threshold_gate( | |
| y, experts["legacy"], experts["patch"], folds, recent, metric | |
| ) | |
| reports.append( | |
| { | |
| "name": f"threshold-{metric}", | |
| "fold_thresholds": thresholds, | |
| "metrics": metrics, | |
| } | |
| ) | |
| print( | |
| f"threshold-{metric}: exact={metrics['exact_match_accuracy']:.4f} " | |
| f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", | |
| flush=True, | |
| ) | |
| entropy_delta = ( | |
| experts["patch"]["confidence"]["negative_entropy"] | |
| - experts["legacy"]["confidence"]["negative_entropy"] | |
| ) | |
| for threshold in (-0.1, -0.05, 0.0, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2): | |
| use_patch = entropy_delta >= threshold | |
| predictions = np.where( | |
| use_patch[:, None], | |
| experts["patch"]["predictions"], | |
| experts["legacy"]["predictions"], | |
| ) | |
| probabilities = np.where( | |
| use_patch[:, None], | |
| experts["patch"]["probabilities"], | |
| experts["legacy"]["probabilities"], | |
| ) | |
| metrics = prediction_metrics(y, probabilities, predictions, recent) | |
| reports.append( | |
| { | |
| "name": f"fixed-negative-entropy-{threshold:g}", | |
| "threshold": threshold, | |
| "patch_fraction": float(use_patch.mean()), | |
| "metrics": metrics, | |
| } | |
| ) | |
| print( | |
| f"fixed-negative-entropy-{threshold:g}: " | |
| f"exact={metrics['exact_match_accuracy']:.4f} " | |
| f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", | |
| flush=True, | |
| ) | |
| for c in (0.001, 0.01, 0.1, 1.0, 10.0): | |
| _, _, metrics = logistic_gate( | |
| y, experts["legacy"], experts["patch"], folds, recent, c | |
| ) | |
| reports.append({"name": f"logistic-C{c:g}", "metrics": metrics}) | |
| print( | |
| f"logistic-C{c:g}: exact={metrics['exact_match_accuracy']:.4f} " | |
| f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", | |
| flush=True, | |
| ) | |
| reports.sort( | |
| key=lambda report: ( | |
| float(report["metrics"]["exact_match_accuracy"]), | |
| float(report["metrics"]["incremental100_exact_match_accuracy"]), | |
| float(report["metrics"]["macro_f1"]), | |
| ), | |
| reverse=True, | |
| ) | |
| output = { | |
| "protocol": "development736 only; every gate prediction is cross-fitted", | |
| "seed": args.seed, | |
| "complementarity": { | |
| "legacy_correct": int(legacy_correct.sum()), | |
| "patch_correct": int(patch_correct.sum()), | |
| "both_correct": int(np.sum(legacy_correct & patch_correct)), | |
| "legacy_only": int(np.sum(legacy_correct & ~patch_correct)), | |
| "patch_only": int(np.sum(~legacy_correct & patch_correct)), | |
| "neither": int(np.sum(~legacy_correct & ~patch_correct)), | |
| "oracle_exact_match_accuracy": float(np.mean(legacy_correct | patch_correct)), | |
| }, | |
| "experts": { | |
| name: {"decoder": expert["decoder"], "metrics": expert["metrics"]} | |
| for name, expert in experts.items() | |
| }, | |
| "gate_candidates": reports, | |
| } | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(reports[0], indent=2), flush=True) | |
| if __name__ == "__main__": | |
| main() | |