Download src/build_v802.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/build_v802.py
- Command line
-
hf download hf://sneo/v802-classifier/src/build_v802.py
-
curl -L -o build_v802.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/build_v802.py
11.3 kB
| #!/usr/bin/env python3 | |
| """Fit the final v802 on development736 and evaluate it once on both holdouts. | |
| The config JSON lists weighted components and the decoder weights chosen on | |
| development OOF (see experiment_v802_ensemble.py): | |
| {"components": [{"name": "v801", "weight": 0.3}, | |
| {"name": "probe", "weight": 0.7, "oof_prefix": "...", | |
| "features": [...], "aug_features": [...], "layers": [...], | |
| "stats": [...], "c": 0.03}], | |
| "prior": 0.5, "card": 0.25} | |
| Component probabilities are fitted on all of development736. The structured | |
| decoder (label-set prior + cardinality model) is fitted on the 296 target | |
| images using stacked OOF probabilities from every dev/check seed, matching how | |
| it was selected. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from scipy.stats import binomtest | |
| from sklearn.linear_model import LogisticRegression | |
| from experiment_v802_probe import l2, pooled_features | |
| from train_v801 import ( | |
| EPS, Spec, all_labelsets, binary_probabilities, fit_binary, labelset_ids, make_variant, | |
| meta_features, report_metrics, | |
| ) | |
| from v802_common import CHECK_SEEDS, DEV_SEEDS, LEGACY_FEATURES, ROOT, dump, load_data, logit | |
| TRIALS = ROOT / "artifacts" / "models" / "v802_trials" | |
| OUTPUT = ROOT / "artifacts" / "models" / "v802" | |
| def knn_predict(x_train, y_train, x_query, k=11, temperature=0.1): | |
| similarity = x_query @ x_train.T | |
| neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:] | |
| neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1) | |
| neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True) | |
| weights = np.exp(neighbour_similarity / temperature) | |
| weights /= weights.sum(axis=1, keepdims=True) | |
| return np.einsum("nk,nkc->nc", weights, y_train[neighbours]) | |
| def fit_component(spec: dict, data, train_idx, query_idx): | |
| """Return (fitted state, query probabilities) for one component.""" | |
| if spec["name"] == "mil": | |
| from experiment_v802_mil import load_projected | |
| from v802_mil import MilConfig, fit_projection, project, train_predict | |
| base = Path(spec["features"]) | |
| x = load_projected(base, spec["layers"], spec["dims"], data) | |
| x_aug = (load_projected(Path(spec["aug_features"]), spec["layers"], spec["dims"], data, | |
| basis_dir=base) if spec.get("aug_features") else None) | |
| # Recompute the (deterministic) PCA bases behind the cached projection so they can be saved. | |
| pca = [] | |
| for slot, layer in enumerate(spec["layers"]): | |
| patches = np.load(base / f"patches_l{layer}.npy", mmap_mode="r") | |
| mean, std, basis = fit_projection(patches, data.anchor_idx, spec["dims"]) | |
| check = project(patches[:4], mean, std, basis) | |
| cached = x[:4, :, slot * spec["dims"] : (slot + 1) * spec["dims"]] | |
| assert np.allclose(check, cached, atol=1e-5), f"PCA mismatch for layer {layer}" | |
| pca.append({"layer": layer, "mean": mean, "std": std, "basis": basis}) | |
| heads: list = [] | |
| probs, _ = train_predict(x, data.y, train_idx, query_idx, MilConfig(**spec["config"]), | |
| spec.get("seed", 802), x_aug, models_out=heads) | |
| return {"mil_config": spec["config"], "layers": spec["layers"], "dims": spec["dims"], | |
| "pca": pca, "heads": heads}, probs | |
| if spec["name"] == "v801": | |
| bundle = np.load(LEGACY_FEATURES) | |
| lookup = {n: i for i, n in enumerate(bundle["file_names"].astype(str))} | |
| order = np.asarray([lookup[n] for n in data.names]) | |
| legacy_x = make_variant(bundle, "legacy")[order] | |
| patch_x = make_variant(bundle, "meanmax_clspatchmax")[order] | |
| probe = fit_binary(legacy_x[train_idx], data.y[train_idx], | |
| Spec("legacy", 0.003, class_weight=None), np.ones(len(train_idx))) | |
| legacy = binary_probabilities(probe, legacy_x[query_idx]) | |
| knn = knn_predict(patch_x[train_idx], data.y[train_idx], patch_x[query_idx]) | |
| state = {"legacy_probe": probe, "knn_train_names": data.names[train_idx]} | |
| return state, 0.8 * legacy + 0.2 * knn | |
| dirs = [Path(d) for d in spec["features"]] | |
| x = l2(np.mean([pooled_features(d, spec["layers"], spec["stats"]) for d in dirs], axis=0)) | |
| views = [x] + [pooled_features(Path(d), spec["layers"], spec["stats"]) | |
| for d in spec.get("aug_features", [])] | |
| probe = fit_binary(np.concatenate([v[train_idx] for v in views]), | |
| np.tile(data.y[train_idx], (len(views), 1)), | |
| Spec("v802probe", spec["c"], class_weight=None), | |
| np.ones(len(train_idx) * len(views))) | |
| probs = np.mean([binary_probabilities(probe, v[query_idx]) for v in views], axis=0) | |
| return {"probe": probe}, probs | |
| def component_oof(spec: dict, data, seed: int) -> np.ndarray: | |
| if spec["name"] == "v801": | |
| stored = np.load(TRIALS / "cache" / f"v801_oof_seed{seed}.npz") | |
| probs = 0.8 * stored["legacy"] + 0.2 * stored["knn"] | |
| else: | |
| stored = np.load(TRIALS / "oof" / f"{spec['oof_prefix']}_seed{seed}.npz") | |
| probs = stored["probs"] | |
| assert (stored["names"] == data.names[data.target_idx]).all() | |
| return probs | |
| def fit_decoder(data, config): | |
| y_target = data.y[data.target_idx] | |
| labelsets = all_labelsets(y_target.shape[1]) | |
| counts = np.bincount(labelset_ids(y_target, labelsets), minlength=len(labelsets)) | |
| log_prior = np.log((counts + 1.0) / (len(y_target) + len(labelsets))) | |
| seeds = [s for s in DEV_SEEDS + CHECK_SEEDS | |
| if all(c["name"] == "v801" or (TRIALS / "oof" / f"{c['oof_prefix']}_seed{s}.npz").exists() | |
| for c in config["components"])] | |
| oof = [sum(c["weight"] * component_oof(c, data, s) for c in config["components"]) for s in seeds] | |
| cardinality = LogisticRegression(C=1.0, max_iter=3000, class_weight="balanced", random_state=801) | |
| cardinality.fit(np.concatenate([meta_features(p) for p in oof]), | |
| np.tile(y_target.sum(axis=1), len(oof))) | |
| return {"labelsets": labelsets, "log_prior": log_prior, "cardinality": cardinality, | |
| "decoder_seeds": seeds} | |
| def decode(decoder, probs, prior, card): | |
| labelsets = decoder["labelsets"] | |
| card_log = np.log(np.clip(decoder["cardinality"].predict_proba(meta_features(probs)), EPS, 1)) | |
| score = logit(probs) @ labelsets.T + prior * decoder["log_prior"] | |
| score += card * card_log[:, labelsets.sum(axis=1).astype(int) - 1] | |
| return labelsets[score.argmax(axis=1)] | |
| def metrics(y, probs, pred): | |
| m = report_metrics(y, probs, pred) | |
| return {"exact_match_accuracy": float(m["exact_match_accuracy"]), | |
| "micro_f1": float(m["micro_f1"]), "macro_f1": float(m["macro_f1"]), | |
| "mean_average_precision": float(m["mean_average_precision"]), | |
| "cardinality_accuracy": float((pred.sum(1) == y.sum(1)).mean()), | |
| "samples": int(len(y))} | |
| def paired(y, pred_new, pred_old, seed=802000, resamples=10000): | |
| new_ok = (pred_new == y).all(1) | |
| old_ok = (pred_old == y).all(1) | |
| gained, lost = int((new_ok & ~old_ok).sum()), int((~new_ok & old_ok).sum()) | |
| rng = np.random.default_rng(seed) | |
| idx = rng.integers(0, len(y), (resamples, len(y))) | |
| deltas = new_ok[idx].mean(1) - old_ok[idx].mean(1) | |
| return {"delta_exact": float(new_ok.mean() - old_ok.mean()), "gained": gained, "lost": lost, | |
| "mcnemar_exact_p": float(binomtest(gained, gained + lost).pvalue) if gained + lost else 1.0, | |
| "bootstrap_delta_95pct": [float(np.quantile(deltas, 0.025)), | |
| float(np.quantile(deltas, 0.975))]} | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, default=OUTPUT) | |
| args = parser.parse_args() | |
| config = json.loads(args.config.read_text()) | |
| dirs = [] | |
| for spec in config["components"]: | |
| for key in ("features", "aug_features"): | |
| value = spec.get(key) or [] | |
| dirs += [value] if isinstance(value, str) else value | |
| names = json.loads((Path(dirs[0]) / "meta.json").read_text())["file_names"] | |
| for d in dirs: | |
| assert json.loads((Path(d) / "meta.json").read_text())["file_names"] == names, d | |
| data = load_data(names) | |
| train_idx = data.development_idx | |
| holdout_idx = np.concatenate((data.new_idx, data.historical_idx)) | |
| decoder = fit_decoder(data, config) # cheap; fails fast on missing/corrupt OOF files | |
| states, holdout_probs, baseline_probs = [], 0.0, None | |
| for spec in config["components"]: | |
| state, probs = fit_component(spec, data, train_idx, holdout_idx) | |
| states.append(state) | |
| holdout_probs = holdout_probs + spec["weight"] * probs | |
| if spec["name"] == "v801": | |
| baseline_probs = probs | |
| pred = decode(decoder, holdout_probs, config["prior"], config["card"]) | |
| if baseline_probs is None: | |
| _, baseline_probs = fit_component({"name": "v801"}, data, train_idx, holdout_idx) | |
| baseline_decoder = fit_decoder(data, {"components": [{"name": "v801", "weight": 1.0}]}) | |
| baseline_pred = decode(baseline_decoder, baseline_probs, 0.5, 0.25) | |
| y = data.y[holdout_idx] | |
| groups = {"new_holdout32": np.arange(len(data.new_idx)), | |
| "historical_holdout32": np.arange(len(data.new_idx), len(holdout_idx)), | |
| "combined64": np.arange(len(holdout_idx))} | |
| report = {"protocol": "v802 components fitted on development736; decoder fitted on stacked " | |
| "target OOF probabilities; each holdout evaluated once. v801 baseline " | |
| "is refitted here with the same protocol (legacy probe + kNN, prior 0.5, " | |
| "card 0.25).", | |
| "config": config, "decoder_seeds": decoder["decoder_seeds"], "holdouts": {}} | |
| for group, rows in groups.items(): | |
| report["holdouts"][group] = { | |
| "v802": metrics(y[rows], holdout_probs[rows], pred[rows]), | |
| "v801_refit": metrics(y[rows], baseline_probs[rows], baseline_pred[rows]), | |
| "paired_v802_vs_v801_refit": paired(y[rows], pred[rows], baseline_pred[rows]), | |
| } | |
| print(group, json.dumps(report["holdouts"][group]["v802"]), | |
| "v801_refit", report["holdouts"][group]["v801_refit"]["exact_match_accuracy"], | |
| flush=True) | |
| classes = list(data.classes) | |
| report["predictions"] = [ | |
| {"file_name": str(data.names[i]), "true": [classes[j] for j in np.flatnonzero(data.y[i])], | |
| "v802": [classes[j] for j in np.flatnonzero(p)], | |
| "v801_refit": [classes[j] for j in np.flatnonzero(b)], | |
| "v802_probabilities": [round(float(v), 6) for v in q]} | |
| for i, p, b, q in zip(holdout_idx, pred, baseline_pred, holdout_probs) | |
| ] | |
| args.output.mkdir(parents=True, exist_ok=True) | |
| dump(args.output / "holdout_metrics.json", report) | |
| joblib.dump({"config": config, "classes": classes, "components": states, "decoder": decoder}, | |
| args.output / "classifier.joblib") | |
| if __name__ == "__main__": | |
| main() | |