| """Swap the shipped decision rule to the cross-fitted winner. |
| |
| Cross-fitted estimate (choose on one half of val, score on the other) says |
| threshold beats the bundle's ef05(0.4) by +0.000808. The PARAMETER is then picked |
| on all of val - legitimate here because we are shipping, not reporting; the honest |
| performance estimate remains the cross-fitted 0.982065. |
| """ |
| import sys, json, shutil |
| import numpy as np, pandas as pd, joblib |
| sys.path.insert(0, "src") |
| import decide as DC |
| from evaluate import macro_f05 |
|
|
| ART = "work/results_A" |
| CACHE = "work/val_scored_20000.parquet" |
| b = joblib.load(f"{ART}/model_bundle.joblib") |
| cand = pd.read_parquet(CACHE) |
| truths = {int(k): set(v) for k, v in json.load(open(CACHE + ".truth.json")).items()} |
| ids = sorted(truths) |
| mm = b["miss_mass"] |
|
|
| def score(method, param): |
| p = DC.predict_sets(cand, "q", "c", "p", method, param, mm, True) |
| m, _ = macro_f05(p, truths, ids) |
| return m["f05"], m["precision"], m["recall"] |
|
|
| grid = np.round(np.arange(0.50, 0.95, 0.01), 3) |
| res = [(score("threshold", v)[0], v) for v in grid] |
| best_f, best_v = max(res) |
| f0, p0, r0 = score(b["method"], b["param"]) |
| f1, p1, r1 = score("threshold", best_v) |
| print(f"current {b['method']}({b['param']}) F0.5 {f0:.6f} P {p0:.4f} R {r0:.4f}") |
| print(f"new threshold({best_v}) F0.5 {f1:.6f} P {p1:.4f} R {r1:.4f}") |
| print(f"delta on full val: {f1-f0:+.6f} (cross-fitted estimate was +0.000808)") |
| print() |
| print("nearby params:", [(v, round(f, 6)) for f, v in sorted(res, reverse=True)[:5]]) |
|
|
| if f1 <= f0: |
| sys.exit("new rule is not better on full val - NOT changing the bundle") |
| shutil.copy(f"{ART}/model_bundle.joblib", f"{ART}/model_bundle.joblib.bak_ef05") |
| b["method"], b["param"] = "threshold", float(best_v) |
| joblib.dump(b, f"{ART}/model_bundle.joblib") |
| print(f"\nbundle updated -> threshold({best_v}) (previous saved as model_bundle.joblib.bak_ef05)") |
|
|
