v802-classifier / src /train_v801_local.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
9.04 kB
#!/usr/bin/env python3
"""Fit the final v801 structured probe with a local DINOv3 correction."""
from __future__ import annotations
import argparse
import csv
import glob
import hashlib
import json
import platform
import time
from pathlib import Path
import joblib
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from experiment_v801_knn import knn_oof
from train_v801 import (
Spec,
all_labelsets,
choose_thresholds,
fit_binary,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
sha256,
spec_dict,
structured_crossfit,
)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--robust-pattern", required=True)
parser.add_argument("--historical-report", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--seed", type=int, default=801208)
args = parser.parse_args()
started = time.time()
bundle = np.load(args.features)
metadata = json.loads(args.features.with_suffix(".json").read_text())
names = bundle["file_names"].astype(str)
y = bundle["labels"].astype(np.uint8)
classes = bundle["classes"]
lookup = {name: index for index, name in enumerate(names)}
with args.development.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
development_idx = np.asarray([lookup[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
if (len(development_idx), len(anchor_idx), len(target_idx)) != (736, 440, 296):
raise ValueError("expected development/anchor/target sizes 736/440/296")
folds = list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=args.seed).split(
np.zeros(len(target_idx)), y[target_idx]
)
)
legacy_x = make_variant(bundle, "legacy")
patch_x = make_variant(bundle, "meanmax_clspatchmax")
spec = Spec("legacy", 0.003, class_weight=None)
legacy_oof = oof_probabilities(
legacy_x, y, origins, anchor_idx, target_idx, folds, spec
)
neighbour_oof = knn_oof(
patch_x, y, anchor_idx, target_idx, folds, k=11, temperature=0.1
)
mixed_oof = 0.8 * legacy_oof + 0.2 * neighbour_oof
decoder = {
"threshold_bias_scale": 0.0,
"labelset_prior_weight": 0.5,
"cardinality_weight": 0.25,
"prior_smoothing": 1.0,
}
predictions = structured_crossfit(
y[target_idx], mixed_oof, folds,
q=decoder["threshold_bias_scale"],
prior_weight=decoder["labelset_prior_weight"],
cardinality_weight=decoder["cardinality_weight"],
)
metrics = report_metrics(y[target_idx], mixed_oof, predictions)
recent = origins[target_idx] == "incremental100"
metrics["incremental100_exact_match_accuracy"] = float(
accuracy_score(y[target_idx][recent], predictions[recent])
)
thresholds = choose_thresholds(y[target_idx], mixed_oof)
labelsets = all_labelsets(y.shape[1])
target_set_ids = labelset_ids(y[target_idx], labelsets)
counts = np.bincount(target_set_ids, minlength=len(labelsets)).astype(np.float64)
labelset_log_prior = np.log((counts + 1.0) / (len(target_idx) + len(labelsets)))
cardinality_model = LogisticRegression(
C=1.0, max_iter=2000, class_weight="balanced", random_state=801
).fit(meta_features(mixed_oof), y[target_idx].sum(axis=1))
probe = fit_binary(
legacy_x[development_idx], y[development_idx], spec,
np.ones(len(development_idx), dtype=np.float64),
)
reference_features = np.asarray(patch_x[development_idx], dtype=np.float32)
reference_labels = np.asarray(y[development_idx], dtype=np.uint8)
reference_digest = hashlib.sha256(reference_features.tobytes()).hexdigest()
artifact = {
"format_version": 3,
"version": "v801",
"method": "DINOv3 structured exact-match probe with local k-NN correction",
"classes": classes,
"components": [
{
"weight": 0.8,
"spec": spec_dict(spec),
"feature_dimension": int(legacy_x.shape[1]),
"scaler": probe["scaler"],
"models": probe["models"],
}
],
"local_correction": {
"weight": 0.2,
"variant": "meanmax_clspatchmax",
"feature_dimension": int(reference_features.shape[1]),
"k": 11,
"temperature": 0.1,
"reference_scope": "development736",
"reference_file_names": names[development_idx],
"reference_features": reference_features,
"reference_labels": reference_labels,
"reference_features_sha256": reference_digest,
},
"thresholds": thresholds,
"labelsets": labelsets,
"labelset_log_prior": labelset_log_prior,
"cardinality_model": cardinality_model,
"decoder": decoder,
"feature_model_path": str(args.model.resolve()),
"feature_model_sha256": metadata["model_sha256"],
"feature_model_file_sha256": metadata["model_file_sha256"],
"feature_type": metadata["feature_type"],
"feature_file_sha256": sha256(args.features),
"descriptor_names": metadata["descriptor_names"],
"min_labels": 1,
"max_labels": 3,
}
robust_reports = []
fixed_name = "q0-prior0.5-card0.25"
for path_text in sorted(glob.glob(args.robust_pattern)):
path = Path(path_text)
report = json.loads(path.read_text())
robust_reports.append(
{
"path": str(path),
"seed": report["seed"],
"metrics": report["fixed_decoders"][fixed_name]["metrics"],
}
)
robust_exact = [
report["metrics"]["exact_match_accuracy"] for report in robust_reports
]
robust_recent = [
report["metrics"]["incremental100_exact_match_accuracy"]
for report in robust_reports
]
historical = json.loads(args.historical_report.read_text())
historical_selected = historical["local_candidates"][fixed_name]["historical_metrics"]
development_report = {
"version": "v801",
"protocol": (
"Final configuration uses development736 only for fitting. Local correction and "
"fixed decoder were selected from four target-domain OOF seeds plus the non-independent "
"historical32 reference. new_holdout32 was not used for this configuration."
),
"samples": {"development": 736, "old_anchors": 440, "target_oof": 296},
"selected": {
"base": {"weight": 0.8, "spec": spec_dict(spec)},
"local_correction": {
"weight": 0.2,
"variant": "meanmax_clspatchmax",
"k": 11,
"temperature": 0.1,
},
"decoder": decoder,
"seed_801208_fixed_decoder_metrics": metrics,
"four_seed_fixed_decoder": {
"mean_exact_match_accuracy": float(np.mean(robust_exact)),
"min_exact_match_accuracy": float(np.min(robust_exact)),
"mean_incremental100_exact_match_accuracy": float(np.mean(robust_recent)),
"reports": robust_reports,
},
"historical_reference_metrics": historical_selected,
"thresholds_fit_on_all_target_oof": thresholds.tolist(),
},
"feature_file_sha256": sha256(args.features),
"reference_features_sha256": reference_digest,
"seed": args.seed,
"seconds": round(time.time() - started, 3),
"python": platform.python_version(),
}
args.output_dir.mkdir(parents=True, exist_ok=True)
joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3)
(args.output_dir / "development_metrics.json").write_text(
json.dumps(development_report, indent=2) + "\n", encoding="utf-8"
)
np.savez_compressed(
args.output_dir / "selected_oof_predictions.npz",
file_names=names[target_idx],
labels=y[target_idx],
probabilities=mixed_oof,
predictions=predictions,
legacy_probabilities=legacy_oof,
neighbour_probabilities=neighbour_oof,
classes=classes,
)
print(json.dumps(development_report["selected"], indent=2), flush=True)
if __name__ == "__main__":
main()