v802-classifier / src /train_v801.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
28.8 kB
#!/usr/bin/env python3
"""Train an exact-match-aware structured probe on frozen DINOv3 features."""
from __future__ import annotations
import argparse
import csv
import hashlib
import itertools
import json
import platform
import time
from dataclasses import dataclass
from pathlib import Path
import joblib
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
accuracy_score,
average_precision_score,
f1_score,
hamming_loss,
roc_auc_score,
)
from sklearn.preprocessing import StandardScaler
EPS = 1e-7
@dataclass(frozen=True)
class Spec:
variant: str
c: float
anchor_weight: float = 1.0
recent_weight: float = 1.0
class_weight: str | None = "balanced"
scale: bool = True
@property
def name(self) -> str:
balance = "bal" if self.class_weight else "plain"
scaled = "std" if self.scale else "raw"
return (
f"{self.variant}-{scaled}-C{self.c:g}-"
f"old{self.anchor_weight:g}-recent{self.recent_weight:g}-{balance}"
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def l2_normalize(x: np.ndarray) -> np.ndarray:
x = np.asarray(x, dtype=np.float32)
return x / np.maximum(np.linalg.norm(x, axis=1, keepdims=True), 1e-12)
def variant_from_descriptors(
descriptors: np.ndarray,
variant: str,
legacy_features: np.ndarray | None = None,
) -> np.ndarray:
descriptors = np.asarray(descriptors, dtype=np.float32)
if descriptors.ndim != 4 or descriptors.shape[1:3] != (4, 4):
raise ValueError(f"unexpected view descriptor shape: {descriptors.shape}")
if variant == "legacy":
if legacy_features is None:
cls_patch = descriptors[:, :, :2].reshape(len(descriptors), 4, -1)
legacy_features = np.concatenate(
(cls_patch.mean(axis=1), cls_patch.max(axis=1)), axis=1
)
return l2_normalize(legacy_features)
descriptor_count = {
"meanmax_clspatchmax": 3,
"meanmax_all": 4,
"concat_clspatch": 2,
"concat_all": 4,
"full_clspatch": 2,
"crops_meanmax_all": 4,
}.get(variant)
if descriptor_count is None:
raise ValueError(f"unknown feature variant: {variant}")
selected = descriptors[:, :, :descriptor_count]
if variant.startswith("meanmax"):
per_view = selected.reshape(len(selected), 4, -1)
x = np.concatenate((per_view.mean(axis=1), per_view.max(axis=1)), axis=1)
elif variant.startswith("concat"):
x = selected.reshape(len(selected), -1)
elif variant == "full_clspatch":
x = selected[:, 0].reshape(len(selected), -1)
else:
crops = selected[:, 1:].reshape(len(selected), 3, -1)
x = np.concatenate((crops.mean(axis=1), crops.max(axis=1)), axis=1)
return l2_normalize(x)
def make_variant(bundle: np.lib.npyio.NpzFile, variant: str) -> np.ndarray:
return variant_from_descriptors(
bundle["view_descriptors"], variant, bundle["features"]
)
def fit_binary(
x: np.ndarray,
y: np.ndarray,
spec: Spec,
sample_weight: np.ndarray,
) -> dict[str, object]:
scaler = StandardScaler().fit(x, sample_weight=sample_weight) if spec.scale else None
z = scaler.transform(x) if scaler is not None else x
models = []
for column in range(y.shape[1]):
model = LogisticRegression(
C=spec.c,
solver="liblinear",
dual=True,
max_iter=3000,
class_weight=spec.class_weight,
random_state=801,
)
model.fit(z, y[:, column], sample_weight=sample_weight)
models.append(model)
return {"scaler": scaler, "models": models}
def binary_probabilities(probe: dict[str, object], x: np.ndarray) -> np.ndarray:
scaler = probe["scaler"]
z = scaler.transform(x) if scaler is not None else x
return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]])
def artifact_probabilities(
artifact: dict[str, object], descriptors: np.ndarray, legacy_features: np.ndarray | None = None
) -> np.ndarray:
result = None
for component in artifact["components"]:
variant = str(component["spec"]["variant"])
x = variant_from_descriptors(descriptors, variant, legacy_features)
if x.shape[1] != int(component["feature_dimension"]):
raise ValueError(f"feature dimension mismatch for {variant}")
probabilities = binary_probabilities(component, x)
weighted = float(component["weight"]) * probabilities
result = weighted if result is None else result + weighted
local = artifact.get("local_correction")
if local is not None:
variant = str(local["variant"])
query = variant_from_descriptors(descriptors, variant, legacy_features)
references = np.asarray(local["reference_features"], dtype=np.float32)
reference_labels = np.asarray(local["reference_labels"], dtype=np.float64)
if query.shape[1] != references.shape[1]:
raise ValueError(f"local feature dimension mismatch for {variant}")
k = int(local["k"])
temperature = float(local["temperature"])
similarity = query @ references.T
neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:]
neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1)
neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True)
weights = np.exp(neighbour_similarity / temperature)
weights /= weights.sum(axis=1, keepdims=True)
probabilities = np.einsum(
"nk,nkc->nc", weights, reference_labels[neighbours]
)
weighted = float(local["weight"]) * probabilities
result = weighted if result is None else result + weighted
if result is None:
raise ValueError("classifier has no components")
return result
def structured_predictions(
artifact: dict[str, object], probabilities: np.ndarray
) -> np.ndarray:
thresholds = np.clip(np.asarray(artifact["thresholds"]), EPS, 1 - EPS)
decoder = artifact["decoder"]
q = float(decoder["threshold_bias_scale"])
logits = np.log(
np.clip(probabilities, EPS, 1 - EPS)
/ np.clip(1 - probabilities, EPS, 1 - EPS)
)
logits -= q * np.log(thresholds / (1 - thresholds))
labelsets = np.asarray(artifact["labelsets"], dtype=np.uint8)
score = logits @ labelsets.T
score += float(decoder["labelset_prior_weight"]) * np.asarray(
artifact["labelset_log_prior"]
)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
card_probabilities = artifact["cardinality_model"].predict_proba(
meta_features(probabilities)
)
score += float(decoder["cardinality_weight"]) * np.log(
np.clip(card_probabilities[:, cardinalities], EPS, 1)
)
return labelsets[score.argmax(axis=1)]
def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray:
grid = np.linspace(0.15, 0.85, 141)
thresholds = []
for column in range(y.shape[1]):
scores = np.asarray([
f1_score(y[:, column], probabilities[:, column] >= threshold)
for threshold in grid
])
best = np.flatnonzero(scores == scores.max())
thresholds.append(grid[best[len(best) // 2]])
return np.asarray(thresholds)
def threshold_predictions(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray:
predictions = (probabilities >= thresholds).astype(np.uint8)
for index, row in enumerate(predictions):
if row.sum() == 0:
predictions[index, np.argmax(probabilities[index] / thresholds)] = 1
elif row.sum() > 3:
positive = np.flatnonzero(row)
keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]]
predictions[index] = 0
predictions[index, keep] = 1
return predictions
def report_metrics(
y: np.ndarray, probabilities: np.ndarray, predictions: np.ndarray
) -> dict[str, object]:
return {
"exact_match_accuracy": float(accuracy_score(y, predictions)),
"micro_f1": float(f1_score(y, predictions, average="micro")),
"macro_f1": float(f1_score(y, predictions, average="macro")),
"per_class_f1": f1_score(y, predictions, average=None).tolist(),
"mean_average_precision": float(average_precision_score(y, probabilities, average="macro")),
"macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")),
"hamming_loss": float(hamming_loss(y, predictions)),
}
def meta_features(probabilities: np.ndarray) -> np.ndarray:
p = np.clip(probabilities, EPS, 1 - EPS)
entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p)).sum(axis=1)
return np.column_stack((p, np.sort(p, axis=1), p.sum(axis=1), p.max(axis=1), entropy))
def all_labelsets(labels: int) -> np.ndarray:
combinations = [
values
for cardinality in (1, 2, 3)
for values in itertools.combinations(range(labels), cardinality)
]
result = np.zeros((len(combinations), labels), dtype=np.uint8)
for index, values in enumerate(combinations):
result[index, list(values)] = 1
return result
def labelset_ids(y: np.ndarray, labelsets: np.ndarray) -> np.ndarray:
lookup = {tuple(row): index for index, row in enumerate(labelsets)}
try:
return np.asarray([lookup[tuple(row)] for row in y], dtype=np.int64)
except KeyError as error:
raise ValueError(f"unsupported labelset: {error.args[0]}") from error
def oof_probabilities(
x: np.ndarray,
y: np.ndarray,
origins: np.ndarray,
anchor_idx: np.ndarray,
target_idx: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
spec: Spec,
) -> np.ndarray:
result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64)
for train_local, valid_local in folds:
train_idx = np.concatenate((anchor_idx, target_idx[train_local]))
weights = np.ones(len(train_idx), dtype=np.float64)
weights[origins[train_idx] == "old440"] = spec.anchor_weight
weights[origins[train_idx] == "incremental100"] = spec.recent_weight
probe = fit_binary(x[train_idx], y[train_idx], spec, weights)
result[valid_local] = binary_probabilities(probe, x[target_idx[valid_local]])
return result
def crossfit_threshold_predictions(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
) -> np.ndarray:
predictions = np.zeros_like(y)
for train, valid in folds:
predictions[valid] = threshold_predictions(
probabilities[valid], choose_thresholds(y[train], probabilities[train])
)
return predictions
def structured_crossfit(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
q: float,
prior_weight: float,
cardinality_weight: float,
smoothing: float = 1.0,
) -> np.ndarray:
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y, labelsets)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
predictions = np.zeros_like(y)
features = meta_features(probabilities)
for train, valid in folds:
thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS)
threshold_bias = q * np.log(thresholds / (1 - thresholds))
p = np.clip(probabilities[valid], EPS, 1 - EPS)
logits = np.log(p / (1 - p)) - threshold_bias
counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64)
log_prior = np.log((counts + smoothing) / (len(train) + smoothing * len(labelsets)))
cardinality = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(features[train], y[train].sum(axis=1))
card_log_probability = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1))
score = logits @ labelsets.T
score += prior_weight * log_prior
score += cardinality_weight * card_log_probability[:, cardinalities]
predictions[valid] = labelsets[score.argmax(axis=1)]
return predictions
def tune_structured_decoder(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
origins: np.ndarray,
) -> tuple[dict[str, float], np.ndarray, dict[str, object]]:
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y, labelsets)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
features = meta_features(probabilities)
contexts = []
for train, valid in folds:
thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS)
threshold_bias = np.log(thresholds / (1 - thresholds))
p = np.clip(probabilities[valid], EPS, 1 - EPS)
logits = np.log(p / (1 - p))
counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64)
log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets)))
cardinality = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(features[train], y[train].sum(axis=1))
card_log_probability = np.log(np.clip(
cardinality.predict_proba(features[valid]), EPS, 1
))
contexts.append((valid, logits, threshold_bias, log_prior, card_log_probability))
best: tuple[tuple[float, ...], dict[str, float], np.ndarray, dict[str, object]] | None = None
recent = origins == "incremental100"
for q in (0.0, 0.25, 0.5, 0.75, 1.0, 1.25):
for prior_weight in (0.0, 0.1, 0.25, 0.5, 0.75, 1.0):
for cardinality_weight in (0.0, 0.25, 0.5, 1.0, 1.5, 2.0):
predictions = np.zeros_like(y)
for valid, logits, threshold_bias, log_prior, card_log_probability in contexts:
score = (logits - q * threshold_bias) @ labelsets.T
score += prior_weight * log_prior
score += cardinality_weight * card_log_probability[:, cardinalities]
predictions[valid] = labelsets[score.argmax(axis=1)]
metrics = report_metrics(y, probabilities, predictions)
recent_exact = float(accuracy_score(y[recent], predictions[recent]))
key = (
float(metrics["exact_match_accuracy"]),
recent_exact,
float(metrics["macro_f1"]),
float(metrics["mean_average_precision"]),
-prior_weight,
-cardinality_weight,
)
params = {
"threshold_bias_scale": q,
"labelset_prior_weight": prior_weight,
"cardinality_weight": cardinality_weight,
"prior_smoothing": 1.0,
}
if best is None or key > best[0]:
metrics["incremental100_exact_match_accuracy"] = recent_exact
best = key, params, predictions, metrics
assert best is not None
return best[1], best[2], best[3]
def spec_dict(spec: Spec) -> dict[str, object]:
return {
"name": spec.name,
"variant": spec.variant,
"c": spec.c,
"anchor_weight": spec.anchor_weight,
"recent_weight": spec.recent_weight,
"class_weight": spec.class_weight,
"scale": spec.scale,
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
started = time.time()
bundle = np.load(args.features)
metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8"))
names = bundle["file_names"].astype(str)
y = bundle["labels"].astype(np.uint8)
classes = bundle["classes"]
name_to_index = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
development_idx = np.asarray([name_to_index[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[name_to_index[row["file_name"]]] = row["origin"]
anchor_idx = np.asarray([index for index in development_idx if origins[index] == "old440"])
target_idx = np.asarray([index for index in development_idx if origins[index] != "old440"])
if (len(development_idx), len(anchor_idx), len(target_idx)) != (736, 440, 296):
raise ValueError("expected development/anchor/target sizes 736/440/296")
folds = list(MultilabelStratifiedKFold(
n_splits=5, shuffle=True, random_state=args.seed
).split(np.zeros(len(target_idx)), y[target_idx]))
variants = (
"legacy",
"meanmax_clspatchmax",
"meanmax_all",
"concat_clspatch",
"concat_all",
"full_clspatch",
"crops_meanmax_all",
)
features_by_variant = {variant: make_variant(bundle, variant) for variant in variants}
oof_cache: dict[Spec, np.ndarray] = {}
simple_reports: list[dict[str, object]] = []
def evaluate_spec(spec: Spec) -> dict[str, object]:
if spec not in oof_cache:
oof_cache[spec] = oof_probabilities(
features_by_variant[spec.variant], y, origins,
anchor_idx, target_idx, folds, spec,
)
probabilities = oof_cache[spec]
predictions = crossfit_threshold_predictions(y[target_idx], probabilities, folds)
metrics = report_metrics(y[target_idx], probabilities, predictions)
recent = origins[target_idx] == "incremental100"
metrics["incremental100_exact_match_accuracy"] = float(
accuracy_score(y[target_idx][recent], predictions[recent])
)
report = {"spec": spec_dict(spec), "crossfit_threshold_metrics": metrics}
print(
f"{spec.name}: exact={metrics['exact_match_accuracy']:.4f} "
f"recent={metrics['incremental100_exact_match_accuracy']:.4f} "
f"macro={metrics['macro_f1']:.4f}",
flush=True,
)
return report
# Stage 1: screen representation families without touching locked manifests.
for variant in variants:
for c in (0.003, 0.01, 0.03):
spec = Spec(variant=variant, c=c)
simple_reports.append(evaluate_spec(spec))
def simple_key(report: dict[str, object]) -> tuple[float, ...]:
metrics = report["crossfit_threshold_metrics"]
return (
float(metrics["exact_match_accuracy"]),
float(metrics["incremental100_exact_match_accuracy"]),
float(metrics["macro_f1"]),
float(metrics["mean_average_precision"]),
)
best_by_variant: dict[str, dict[str, object]] = {}
for report in simple_reports:
variant = str(report["spec"]["variant"])
if variant not in best_by_variant or simple_key(report) > simple_key(best_by_variant[variant]):
best_by_variant[variant] = report
top_variants = [
str(report["spec"]["variant"])
for report in sorted(best_by_variant.values(), key=simple_key, reverse=True)[:2]
]
print(f"Refining variants: {top_variants}", flush=True)
# Stage 2: tune regularization, old-domain influence, and class balancing.
for variant in top_variants:
for c in (0.003, 0.01, 0.03, 0.1):
for anchor_weight in (0.25, 0.5, 1.0):
for class_weight in ("balanced", None):
spec = Spec(
variant=variant,
c=c,
anchor_weight=anchor_weight,
class_weight=class_weight,
)
if spec not in oof_cache:
simple_reports.append(evaluate_spec(spec))
# A small, targeted domain-adaptation sweep for the primary evaluation domain.
for c in (0.01, 0.03):
for anchor_weight in (0.5, 1.0):
for class_weight in ("balanced", None):
spec = Spec(
variant=variant,
c=c,
anchor_weight=anchor_weight,
recent_weight=2.0,
class_weight=class_weight,
)
if spec not in oof_cache:
simple_reports.append(evaluate_spec(spec))
simple_reports.sort(key=simple_key, reverse=True)
structured_reports: list[dict[str, object]] = []
# Keep decoder search bounded and include distinct probability models.
seen: set[Spec] = set()
finalists: list[Spec] = []
for report in simple_reports:
spec_values = report["spec"]
spec = Spec(
variant=str(spec_values["variant"]),
c=float(spec_values["c"]),
anchor_weight=float(spec_values["anchor_weight"]),
recent_weight=float(spec_values["recent_weight"]),
class_weight=spec_values["class_weight"],
scale=bool(spec_values["scale"]),
)
if spec not in seen:
finalists.append(spec)
seen.add(spec)
if len(finalists) == 8:
break
for spec in finalists:
params, predictions, metrics = tune_structured_decoder(
y[target_idx], oof_cache[spec], folds, origins[target_idx]
)
structured_reports.append({
"components": [{"weight": 1.0, "spec": spec_dict(spec)}],
"decoder": params,
"metrics": metrics,
"probabilities": oof_cache[spec],
"predictions": predictions,
"specs": (spec,),
"weights": (1.0,),
})
print(
f"structured {spec.name}: exact={metrics['exact_match_accuracy']:.4f} "
f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True
)
# Stage 3: probability ensembles reduce fold/representation variance.
ensemble_pool = finalists[:4]
for first, second in itertools.combinations(ensemble_pool, 2):
for first_weight in (0.25, 0.5, 0.75):
probabilities = (
first_weight * oof_cache[first] + (1 - first_weight) * oof_cache[second]
)
params, predictions, metrics = tune_structured_decoder(
y[target_idx], probabilities, folds, origins[target_idx]
)
structured_reports.append({
"components": [
{"weight": first_weight, "spec": spec_dict(first)},
{"weight": 1 - first_weight, "spec": spec_dict(second)},
],
"decoder": params,
"metrics": metrics,
"probabilities": probabilities,
"predictions": predictions,
"specs": (first, second),
"weights": (first_weight, 1 - first_weight),
})
def structured_key(report: dict[str, object]) -> tuple[float, ...]:
metrics = report["metrics"]
return (
float(metrics["exact_match_accuracy"]),
float(metrics["incremental100_exact_match_accuracy"]),
float(metrics["macro_f1"]),
float(metrics["mean_average_precision"]),
-float(len(report["components"])),
)
structured_reports.sort(key=structured_key, reverse=True)
selected = structured_reports[0]
selected_probabilities = selected["probabilities"]
selected_predictions = selected["predictions"]
thresholds = choose_thresholds(y[target_idx], selected_probabilities)
labelsets = all_labelsets(y.shape[1])
target_set_ids = labelset_ids(y[target_idx], labelsets)
smoothing = float(selected["decoder"]["prior_smoothing"])
counts = np.bincount(target_set_ids, minlength=len(labelsets)).astype(np.float64)
labelset_log_prior = np.log(
(counts + smoothing) / (len(target_idx) + smoothing * len(labelsets))
)
cardinality_model = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(meta_features(selected_probabilities), y[target_idx].sum(axis=1))
final_components = []
for spec, weight in zip(selected["specs"], selected["weights"]):
weights = np.ones(len(development_idx), dtype=np.float64)
weights[origins[development_idx] == "old440"] = spec.anchor_weight
weights[origins[development_idx] == "incremental100"] = spec.recent_weight
probe = fit_binary(
features_by_variant[spec.variant][development_idx],
y[development_idx], spec, weights,
)
final_components.append({
"weight": weight,
"spec": spec_dict(spec),
"feature_dimension": int(features_by_variant[spec.variant].shape[1]),
"scaler": probe["scaler"],
"models": probe["models"],
})
artifact = {
"format_version": 2,
"version": "v801",
"method": "DINOv3 structured multi-view exact-match probe",
"classes": classes,
"components": final_components,
"thresholds": thresholds,
"labelsets": labelsets,
"labelset_log_prior": labelset_log_prior,
"cardinality_model": cardinality_model,
"decoder": selected["decoder"],
"feature_model_path": str(args.model.resolve()),
"feature_model_sha256": metadata["model_sha256"],
"feature_model_file_sha256": metadata["model_file_sha256"],
"feature_type": metadata["feature_type"],
"feature_file_sha256": sha256(args.features),
"descriptor_names": metadata["descriptor_names"],
"min_labels": 1,
"max_labels": 3,
}
args.output_dir.mkdir(parents=True, exist_ok=True)
joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3)
np.savez_compressed(
args.output_dir / "selected_oof_predictions.npz",
file_names=names[target_idx],
labels=y[target_idx],
probabilities=selected_probabilities,
predictions=selected_predictions,
classes=classes,
)
compact_structured = []
for report in structured_reports:
compact_structured.append({
"components": report["components"],
"decoder": report["decoder"],
"metrics": report["metrics"],
})
development_report = {
"version": "v801",
"protocol": (
"Five-fold multilabel-stratified target-domain OOF selection on 296 images; "
"440 old-domain anchors remain train-only in every fold. Feature conditioning, "
"old-domain sample weight, binary-head regularization, class balancing, labelset "
"prior, cardinality model, and probability ensembling are selected without reading "
"either locked evaluation manifest. Cross-fit threshold/cardinality predictions "
"are development-selection estimates, not final unbiased test estimates."
),
"samples": {"development": 736, "old_anchors": 440, "target_oof": 296},
"selected": {
"components": selected["components"],
"decoder": selected["decoder"],
"metrics": selected["metrics"],
"thresholds_fit_on_all_target_oof": thresholds.tolist(),
},
"structured_candidates": compact_structured,
"simple_candidates": simple_reports,
"feature_dimensions": {
variant: int(values.shape[1]) for variant, values in features_by_variant.items()
},
"classes": classes.tolist(),
"seed": args.seed,
"seconds": round(time.time() - started, 3),
"python": platform.python_version(),
"feature_file_sha256": sha256(args.features),
}
(args.output_dir / "development_metrics.json").write_text(
json.dumps(development_report, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(json.dumps(development_report["selected"], ensure_ascii=False, indent=2), flush=True)
if __name__ == "__main__":
main()