| """Singletons are the largest unexamined loss. Is any of it recoverable? |
| |
| From the recovered metrics.json of the submitted run (n=50,000 val entities): |
| f05_singletons 0.9438 |
| f05_non_singletons 0.9834 |
| singleton_share 0.05484 |
| A singleton scores 1.0 for an empty prediction and 0.0 for ANY prediction, so 0.9438 means |
| 5.62% of true singletons were handed at least one confident false positive. Macro cost: |
| 0.05484 * 0.0562 = 0.00308 |
| which is larger than the corroboration, second-cross-encoder and per-country-threshold levers |
| combined. FINAL_SOLUTION.md s32 item 7 names this exact failure and s29 then EXCLUDES a singleton |
| classifier; this script tests whether that exclusion was justified. |
| |
| Two entity-level failures exist and both score exactly 0: |
| A true singleton, we predicted something |
| B true non-singleton, we predicted nothing |
| Fixing A by raising the bar necessarily creates more B. The question is whether the two |
| populations are SEPARABLE by anything we can compute without labels. |
| """ |
| import json, sys |
| import numpy as np, pandas as pd, joblib |
| sys.path.insert(0, "src") |
| import decide as DC |
| from evaluate import macro_f05 |
|
|
| ART, CACHE = "work/results_A", "work/val_scored_20000.parquet" |
| b = joblib.load(f"{ART}/model_bundle.joblib") |
| MM, THR = b["miss_mass"], b["param"] |
| cand = pd.read_parquet(CACHE) |
| truths = {int(k): set(v) for k, v in json.load(open(CACHE + ".truth.json")).items()} |
| ids = sorted(truths) |
| shipped = DC.predict_sets(cand, "q", "c", "p", b["method"], THR, MM, True) |
| m0, per = macro_f05(shipped, truths, ids) |
| print(f"baseline F0.5 {m0['f05']:.5f} (rule {b['method']}({THR}))") |
|
|
| is_sing = {q: (len(truths[q]) == 0) for q in ids} |
| n_sing = sum(is_sing.values()) |
| print(f"\nval entities {len(ids):,} true singletons {n_sing:,} ({n_sing/len(ids):.3%})") |
|
|
| sing_ids = [q for q in ids if is_sing[q]] |
| non_ids = [q for q in ids if not is_sing[q]] |
| ms, _ = macro_f05({q: shipped.get(q, []) for q in sing_ids}, truths, sing_ids) |
| mn, _ = macro_f05({q: shipped.get(q, []) for q in non_ids}, truths, non_ids) |
| print(f" F0.5 singletons {ms['f05']:.5f} ({n_sing:,} entities)") |
| print(f" F0.5 non-singletons {mn['f05']:.5f} ({len(non_ids):,} entities)") |
| share = n_sing / len(ids) |
| print(f" macro cost of singleton errors : {share*(1-ms['f05']):.5f}") |
| print(f" macro cost of non-sing errors : {(1-share)*(1-mn['f05']):.5f}") |
|
|
| |
| A = [q for q in sing_ids if shipped.get(q)] |
| B = [q for q in non_ids if not shipped.get(q)] |
| print(f"\nclass A true singleton but predicted : {len(A):,} / {n_sing:,} " |
| f"({len(A)/max(n_sing,1):.2%})") |
| print(f"class B non-singleton but empty : {len(B):,} / {len(non_ids):,} " |
| f"({len(B)/max(len(non_ids),1):.2%})") |
| print(f" each costs a full 1.0; together {(len(A)+len(B))/len(ids):.4%} of the macro average") |
|
|
| by_q = {q: g for q, g in cand.groupby("q")} |
|
|
| def profile(qs, label): |
| """Label-free signals available at inference time.""" |
| rows = [] |
| for q in qs: |
| g = by_q.get(q) |
| if g is None or not len(g): |
| continue |
| p = g["p"].to_numpy() |
| srt = np.sort(p)[::-1] |
| rows.append({ |
| "n_cand": len(p), |
| "top1": srt[0], |
| "top2": srt[1] if len(srt) > 1 else np.nan, |
| "margin": srt[0] - (srt[1] if len(srt) > 1 else 0.0), |
| "n_above_thr": int((p >= THR).sum()), |
| "psum": float(p.sum()), |
| "mean_p": float(p.mean()), |
| }) |
| d = pd.DataFrame(rows) |
| print(f"\n--- {label} n={len(d):,} ---") |
| if not len(d): |
| return d |
| print(d[["n_cand", "top1", "top2", "margin", "n_above_thr", "psum", "mean_p"]] |
| .describe().loc[["mean", "50%"]].to_string()) |
| return d |
|
|
| dA = profile(A, "A: true singleton, we predicted (the 1.0 losses)") |
| dS_ok = profile([q for q in sing_ids if not shipped.get(q)], |
| "singletons we correctly left empty") |
| dN_ok = profile([q for q in non_ids if shipped.get(q)], |
| "non-singletons we predicted (the good case)") |
| dB = profile(B, "B: non-singleton, we predicted nothing") |
|
|
| print("\n=== ARE CLASS-A ERRORS SEPARABLE FROM CORRECT NON-SINGLETON PREDICTIONS? ===") |
| print("If class A had systematically lower top1 / psum, a higher bar for weak entities would") |
| print("trade A-errors for B-errors profitably. Compare the two distributions:") |
| for col in ("top1", "margin", "psum", "n_above_thr", "n_cand"): |
| a = dA[col].dropna() if len(dA) else pd.Series(dtype=float) |
| n = dN_ok[col].dropna() if len(dN_ok) else pd.Series(dtype=float) |
| if not len(a) or not len(n): |
| continue |
| |
| q10 = np.percentile(n, 10) |
| print(f" {col:<12} A median {a.median():.4f} good median {n.median():.4f} " |
| f"A above good-p10 ({q10:.4f}): {(a >= q10).mean():.1%}") |
|
|
| print("\n=== DIRECT TEST: raise the bar and count the actual trade ===") |
| print("Requires top1 >= tau before any prediction is allowed for that entity.") |
| print(f"{'tau':>7}{'A fixed':>10}{'B created':>11}{'net entities':>14}{'F0.5':>11}{'delta':>10}") |
| for tau in [0.70, 0.80, 0.90, 0.95, 0.98, 0.99, 0.995, 0.999]: |
| pr = {} |
| for q, cs in shipped.items(): |
| g = by_q.get(q) |
| if g is None: |
| continue |
| if float(g["p"].max()) >= tau: |
| pr[q] = cs |
| m, _ = macro_f05(pr, truths, ids) |
| a_fixed = sum(1 for q in A if q not in pr) |
| b_new = sum(1 for q in non_ids if shipped.get(q) and q not in pr) |
| print(f"{tau:>7.3f}{a_fixed:>10,}{b_new:>11,}{a_fixed-b_new:>14,}" |
| f"{m['f05']:>11.5f}{m['f05']-m0['f05']:>+10.5f}") |
|
|
| print("\n=== CEILING: what a PERFECT singleton detector would be worth ===") |
| perfect = {q: cs for q, cs in shipped.items() if not is_sing[q]} |
| m, _ = macro_f05(perfect, truths, ids) |
| print(f" drop every prediction on a TRUE singleton (oracle): {m['f05']:.5f} " |
| f"({m['f05']-m0['f05']:+.5f})") |
| print(f" that is the hard ceiling on all singleton work.") |
|
|
