"""Swap the shipped decision rule to the cross-fitted winner. Cross-fitted estimate (choose on one half of val, score on the other) says threshold beats the bundle's ef05(0.4) by +0.000808. The PARAMETER is then picked on all of val - legitimate here because we are shipping, not reporting; the honest performance estimate remains the cross-fitted 0.982065. """ import sys, json, shutil import numpy as np, pandas as pd, joblib sys.path.insert(0, "src") import decide as DC from evaluate import macro_f05 ART = "work/results_A" CACHE = "work/val_scored_20000.parquet" b = joblib.load(f"{ART}/model_bundle.joblib") cand = pd.read_parquet(CACHE) truths = {int(k): set(v) for k, v in json.load(open(CACHE + ".truth.json")).items()} ids = sorted(truths) mm = b["miss_mass"] def score(method, param): p = DC.predict_sets(cand, "q", "c", "p", method, param, mm, True) m, _ = macro_f05(p, truths, ids) return m["f05"], m["precision"], m["recall"] grid = np.round(np.arange(0.50, 0.95, 0.01), 3) res = [(score("threshold", v)[0], v) for v in grid] best_f, best_v = max(res) f0, p0, r0 = score(b["method"], b["param"]) f1, p1, r1 = score("threshold", best_v) print(f"current {b['method']}({b['param']}) F0.5 {f0:.6f} P {p0:.4f} R {r0:.4f}") print(f"new threshold({best_v}) F0.5 {f1:.6f} P {p1:.4f} R {r1:.4f}") print(f"delta on full val: {f1-f0:+.6f} (cross-fitted estimate was +0.000808)") print() print("nearby params:", [(v, round(f, 6)) for f, v in sorted(res, reverse=True)[:5]]) if f1 <= f0: sys.exit("new rule is not better on full val - NOT changing the bundle") shutil.copy(f"{ART}/model_bundle.joblib", f"{ART}/model_bundle.joblib.bak_ef05") b["method"], b["param"] = "threshold", float(best_v) joblib.dump(b, f"{ART}/model_bundle.joblib") print(f"\nbundle updated -> threshold({best_v}) (previous saved as model_bundle.joblib.bak_ef05)")