v802-classifier / src /experiment_v801_gate.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
13.5 kB
#!/usr/bin/env python3
"""Cross-fitted confidence gates between complementary v801 experts."""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from scipy.special import logsumexp
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
from train_v801 import (
EPS,
Spec,
all_labelsets,
choose_thresholds,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
tune_structured_decoder,
)
def expert_outputs(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
origins: np.ndarray,
) -> dict[str, object]:
decoder, tuned_predictions, metrics = tune_structured_decoder(
y, probabilities, folds, origins
)
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y, labelsets)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
predictions = np.zeros_like(y)
confidence = {
name: np.zeros(len(y), dtype=np.float64)
for name in (
"joint_margin",
"joint_top_probability",
"bernoulli_set_score",
"negative_entropy",
"mean_binary_margin",
"min_binary_margin",
"cardinality_confidence",
)
}
p = np.clip(probabilities, EPS, 1 - EPS)
binary_entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p))
for train, valid in folds:
thresholds = np.clip(choose_thresholds(y[train], p[train]), EPS, 1 - EPS)
logits = np.log(p[valid] / (1 - p[valid]))
logits -= float(decoder["threshold_bias_scale"]) * np.log(
thresholds / (1 - thresholds)
)
counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64)
smoothing = float(decoder["prior_smoothing"])
log_prior = np.log(
(counts + smoothing) / (len(train) + smoothing * len(labelsets))
)
cardinality = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(meta_features(p[train]), y[train].sum(axis=1))
card_probability = np.clip(
cardinality.predict_proba(meta_features(p[valid])), EPS, 1
)
score = logits @ labelsets.T
score += float(decoder["labelset_prior_weight"]) * log_prior
score += float(decoder["cardinality_weight"]) * np.log(
card_probability[:, cardinalities]
)
order = np.argsort(score, axis=1)
best = order[:, -1]
predictions[valid] = labelsets[best]
confidence["joint_margin"][valid] = score[np.arange(len(valid)), order[:, -1]] - score[
np.arange(len(valid)), order[:, -2]
]
confidence["joint_top_probability"][valid] = np.exp(
score[np.arange(len(valid)), best] - logsumexp(score, axis=1)
)
selected = labelsets[best]
confidence["bernoulli_set_score"][valid] = (
selected * np.log(p[valid]) + (1 - selected) * np.log(1 - p[valid])
).sum(axis=1)
confidence["negative_entropy"][valid] = -binary_entropy[valid].sum(axis=1)
margin = np.abs(p[valid] - thresholds)
confidence["mean_binary_margin"][valid] = margin.mean(axis=1)
confidence["min_binary_margin"][valid] = margin.min(axis=1)
confidence["cardinality_confidence"][valid] = card_probability.max(axis=1)
if not np.array_equal(predictions, tuned_predictions):
raise RuntimeError("expert score reconstruction does not match tuned predictions")
return {
"decoder": decoder,
"predictions": predictions,
"probabilities": probabilities,
"metrics": metrics,
"confidence": confidence,
}
def prediction_metrics(
y: np.ndarray,
probabilities: np.ndarray,
predictions: np.ndarray,
recent: np.ndarray,
) -> dict[str, object]:
metrics = report_metrics(y, probabilities, predictions)
metrics["incremental100_exact_match_accuracy"] = float(
accuracy_score(y[recent], predictions[recent])
)
return metrics
def threshold_gate(
y: np.ndarray,
legacy: dict[str, object],
patch: dict[str, object],
folds: list[tuple[np.ndarray, np.ndarray]],
recent: np.ndarray,
metric: str,
) -> tuple[np.ndarray, np.ndarray, list[float], dict[str, object]]:
delta = patch["confidence"][metric] - legacy["confidence"][metric]
legacy_correct = np.all(legacy["predictions"] == y, axis=1)
patch_correct = np.all(patch["predictions"] == y, axis=1)
use_patch = np.zeros(len(y), dtype=bool)
thresholds = []
for train, valid in folds:
values = np.unique(delta[train])
candidates = np.concatenate(
((-np.inf,), (values[:-1] + values[1:]) / 2, (np.inf,))
)
best = None
for threshold in candidates:
choose_patch = delta[train] >= threshold
correct = np.where(choose_patch, patch_correct[train], legacy_correct[train])
key = (
float(correct.mean()),
float(correct[recent[train]].mean()) if np.any(recent[train]) else 0.0,
-abs(float(threshold)) if np.isfinite(threshold) else -1e12,
)
if best is None or key > best[0]:
best = key, float(threshold)
assert best is not None
thresholds.append(best[1])
use_patch[valid] = delta[valid] >= best[1]
predictions = np.where(
use_patch[:, None], patch["predictions"], legacy["predictions"]
)
probabilities = np.where(
use_patch[:, None], patch["probabilities"], legacy["probabilities"]
)
return predictions, probabilities, thresholds, prediction_metrics(
y, probabilities, predictions, recent
)
def logistic_gate(
y: np.ndarray,
legacy: dict[str, object],
patch: dict[str, object],
folds: list[tuple[np.ndarray, np.ndarray]],
recent: np.ndarray,
c: float,
) -> tuple[np.ndarray, np.ndarray, dict[str, object]]:
confidence_names = sorted(legacy["confidence"])
confidence_delta = np.column_stack(
[patch["confidence"][name] - legacy["confidence"][name] for name in confidence_names]
)
x = np.column_stack(
(
confidence_delta,
patch["probabilities"] - legacy["probabilities"],
patch["predictions"].astype(np.float64)
- legacy["predictions"].astype(np.float64),
patch["predictions"].sum(axis=1).astype(np.float64)
- legacy["predictions"].sum(axis=1).astype(np.float64),
)
)
legacy_correct = np.all(legacy["predictions"] == y, axis=1)
patch_correct = np.all(patch["predictions"] == y, axis=1)
discordant = legacy_correct != patch_correct
target = patch_correct.astype(np.uint8)
use_patch = np.ones(len(y), dtype=bool)
for train, valid in folds:
gate_train = train[discordant[train]]
scaler = StandardScaler().fit(x[gate_train])
model = LogisticRegression(
C=c,
solver="liblinear",
class_weight="balanced",
random_state=801,
).fit(scaler.transform(x[gate_train]), target[gate_train])
use_patch[valid] = model.predict_proba(scaler.transform(x[valid]))[:, 1] >= 0.5
predictions = np.where(
use_patch[:, None], patch["predictions"], legacy["predictions"]
)
probabilities = np.where(
use_patch[:, None], patch["probabilities"], legacy["probabilities"]
)
return predictions, probabilities, prediction_metrics(
y, probabilities, predictions, recent
)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
bundle = np.load(args.features)
names = bundle["file_names"].astype(str)
y_all = bundle["labels"].astype(np.uint8)
lookup = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
development_idx = np.asarray([lookup[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
y = y_all[target_idx]
target_origins = origins[target_idx]
recent = target_origins == "incremental100"
folds = list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split(
np.zeros(len(target_idx)), y
)
)
specs = {
"legacy": Spec("legacy", 0.003, class_weight=None),
"patch": Spec("meanmax_clspatchmax", 0.003, class_weight=None),
}
experts = {}
for name, spec in specs.items():
probabilities = oof_probabilities(
make_variant(bundle, spec.variant), y_all, origins,
anchor_idx, target_idx, folds, spec,
)
experts[name] = expert_outputs(y, probabilities, folds, target_origins)
print(
f"{name}: {experts[name]['metrics']['exact_match_accuracy']:.4f}",
flush=True,
)
legacy_correct = np.all(experts["legacy"]["predictions"] == y, axis=1)
patch_correct = np.all(experts["patch"]["predictions"] == y, axis=1)
reports = []
for metric in sorted(experts["legacy"]["confidence"]):
predictions, probabilities, thresholds, metrics = threshold_gate(
y, experts["legacy"], experts["patch"], folds, recent, metric
)
reports.append(
{
"name": f"threshold-{metric}",
"fold_thresholds": thresholds,
"metrics": metrics,
}
)
print(
f"threshold-{metric}: exact={metrics['exact_match_accuracy']:.4f} "
f"recent={metrics['incremental100_exact_match_accuracy']:.4f}",
flush=True,
)
entropy_delta = (
experts["patch"]["confidence"]["negative_entropy"]
- experts["legacy"]["confidence"]["negative_entropy"]
)
for threshold in (-0.1, -0.05, 0.0, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2):
use_patch = entropy_delta >= threshold
predictions = np.where(
use_patch[:, None],
experts["patch"]["predictions"],
experts["legacy"]["predictions"],
)
probabilities = np.where(
use_patch[:, None],
experts["patch"]["probabilities"],
experts["legacy"]["probabilities"],
)
metrics = prediction_metrics(y, probabilities, predictions, recent)
reports.append(
{
"name": f"fixed-negative-entropy-{threshold:g}",
"threshold": threshold,
"patch_fraction": float(use_patch.mean()),
"metrics": metrics,
}
)
print(
f"fixed-negative-entropy-{threshold:g}: "
f"exact={metrics['exact_match_accuracy']:.4f} "
f"recent={metrics['incremental100_exact_match_accuracy']:.4f}",
flush=True,
)
for c in (0.001, 0.01, 0.1, 1.0, 10.0):
_, _, metrics = logistic_gate(
y, experts["legacy"], experts["patch"], folds, recent, c
)
reports.append({"name": f"logistic-C{c:g}", "metrics": metrics})
print(
f"logistic-C{c:g}: exact={metrics['exact_match_accuracy']:.4f} "
f"recent={metrics['incremental100_exact_match_accuracy']:.4f}",
flush=True,
)
reports.sort(
key=lambda report: (
float(report["metrics"]["exact_match_accuracy"]),
float(report["metrics"]["incremental100_exact_match_accuracy"]),
float(report["metrics"]["macro_f1"]),
),
reverse=True,
)
output = {
"protocol": "development736 only; every gate prediction is cross-fitted",
"seed": args.seed,
"complementarity": {
"legacy_correct": int(legacy_correct.sum()),
"patch_correct": int(patch_correct.sum()),
"both_correct": int(np.sum(legacy_correct & patch_correct)),
"legacy_only": int(np.sum(legacy_correct & ~patch_correct)),
"patch_only": int(np.sum(~legacy_correct & patch_correct)),
"neither": int(np.sum(~legacy_correct & ~patch_correct)),
"oracle_exact_match_accuracy": float(np.mean(legacy_correct | patch_correct)),
},
"experts": {
name: {"decoder": expert["decoder"], "metrics": expert["metrics"]}
for name, expert in experts.items()
},
"gate_candidates": reports,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
print(json.dumps(reports[0], indent=2), flush=True)
if __name__ == "__main__":
main()