| """The 1,763 true pairs we RETRIEVED but REJECTED: are they rankable, or hopeless? |
| |
| If their calibrated p sits just under the 0.70 threshold, a better-calibrated or |
| slightly better model recovers them cheaply. If p ~ 0, the reranker genuinely cannot |
| tell them from negatives and only a different model/feature would help. |
| |
| Also measures, exactly: |
| - how many FPs we would buy per TP recovered at each threshold band (the real |
| precision/recall exchange rate, which is what F0.5 actually cares about) |
| - the rank of the rejected true pair within its entity's candidate list |
| - whether one-owner arbitration is stealing any of them |
| """ |
| import json, sys |
| import numpy as np, pandas as pd, joblib |
| sys.path.insert(0, "src") |
| import decide as DC |
| from evaluate import macro_f05 |
|
|
| ART, CACHE = "work/results_A", "work/val_scored_20000.parquet" |
| b = joblib.load(f"{ART}/model_bundle.joblib") |
| mm = b["miss_mass"] |
| cand = pd.read_parquet(CACHE) |
| truths = {int(k): set(v) for k, v in json.load(open(CACHE + ".truth.json")).items()} |
| ids = sorted(truths) |
| THR = b["param"] |
| print(f"shipped: {b['method']}({THR}) miss_mass={mm:.4f}") |
|
|
| shipped = DC.predict_sets(cand, "q", "c", "p", b["method"], THR, mm, True) |
| m0, _ = macro_f05(shipped, truths, ids) |
| print(f"shipped F0.5 {m0['f05']:.5f} P {m0['precision']:.4f} R {m0['recall']:.4f}\n") |
|
|
| |
| tset = {q: truths[q] for q in ids} |
| cand = cand.copy() |
| cand["y"] = [1 if c in tset.get(q, ()) else 0 for q, c in zip(cand["q"], cand["c"])] |
| pred_pairs = {(q, c) for q, cs in shipped.items() for c in cs} |
| cand["sel"] = [1 if (q, c) in pred_pairs else 0 for q, c in zip(cand["q"], cand["c"])] |
|
|
| rej = cand[(cand.y == 1) & (cand.sel == 0)] |
| sel = cand[cand.sel == 1] |
| print(f"retrieved-but-rejected true pairs : {len(rej):,}") |
| print(f"selected pairs : {len(sel):,} (of which false {int((sel.y==0).sum()):,})") |
|
|
| print("\n=== p-distribution of the REJECTED true pairs ===") |
| bins = [0,.01,.05,.1,.2,.3,.4,.5,.6,.65,.70,.8,.9,1.01] |
| h, _ = np.histogram(rej["p"], bins=bins) |
| cum = 0 |
| for lo, hi, c in zip(bins[:-1], bins[1:], h): |
| cum += c |
| print(f" p in [{lo:.2f},{hi:.2f}) : {c:>6,} ({c/len(rej):6.2%}) cum {cum/len(rej):6.2%}") |
| print(f" median p of a rejected true pair: {rej['p'].median():.4f}") |
| print(f" ** fraction with p >= 0.5 (near-misses, cheap to recover): {(rej['p']>=0.5).mean():.2%}") |
| print(f" ** fraction with p < 0.05 (model is confidently wrong) : {(rej['p']<0.05).mean():.2%}") |
|
|
| print("\n=== EXCHANGE RATE: what each threshold step actually buys ===") |
| print(f"{'band':>14} {'TP gained':>10} {'FP gained':>10} {'ratio':>8}") |
| unsel = cand[cand.sel == 0] |
| for lo, hi in [(0.65,0.70),(0.60,0.65),(0.50,0.60),(0.40,0.50),(0.30,0.40),(0.10,0.30),(0.0,0.10)]: |
| s = unsel[(unsel.p >= lo) & (unsel.p < hi)] |
| tp, fp = int((s.y==1).sum()), int((s.y==0).sum()) |
| r = f"{tp/fp:.2f}" if fp else "inf" |
| print(f" [{lo:.2f},{hi:.2f}) {tp:>10,} {fp:>10,} {r:>8}") |
| print(" (F0.5 rises only while a band's TP:FP beats roughly 1:4 at our operating point)") |
|
|
| print("\n=== rank of the rejected true pair inside its entity ===") |
| cand["rk"] = cand.groupby("q")["p"].rank(ascending=False, method="first") |
| rj = cand[(cand.y==1) & (cand.sel==0)] |
| for k in [1,2,3,4,5,8,12]: |
| print(f" rank <= {k:<3}: {(rj.rk<=k).mean():6.2%}") |
|
|
| print("\n=== is one-owner arbitration stealing true pairs? ===") |
| d = DC.resolve_conflicts(cand, "p") |
| kept = {(q,c) for q,c in zip(d["q"], d["c"])} |
| lost = cand[(cand.y==1) & (~pd.Series(list(zip(cand['q'],cand['c']))).isin(kept).values)] |
| print(f" true pairs dropped by arbitration: {len(lost):,}") |
|
|
| print("\n=== ORACLE-RERANK: if we could perfectly reorder within each entity, ===") |
| print("=== keeping the SAME number of predictions per entity (no precision cost) ===") |
| ns = {q: len(cs) for q, cs in shipped.items()} |
| orc = {} |
| for q, g in cand.groupby("q"): |
| k = ns.get(q, 0) |
| if k: |
| g2 = g.sort_values("y", ascending=False, kind="mergesort") |
| orc[q] = list(g2["c"].values[:k]) |
| m1, _ = macro_f05(orc, truths, ids) |
| print(f" perfect within-entity ranking, same k : F0.5 {m1['f05']:.5f} (+{m1['f05']-m0['f05']:.5f})") |
| print(" ^ this is what a STRICTLY BETTER RERANKER alone can win, holding k fixed") |
|
|
