model-3/scripts/structure.py

"""Is there exploitable CARDINALITY STRUCTURE the flat threshold ignores?

Our decision layer is one global threshold on calibrated probability. It knows nothing about how
many matches an entity is likely to have, nor how they split across S2 and S3. If the ground
truth is strongly structured - say most non-singletons are exactly one S2 plus one S3 - then k is
partly determined by structure rather than probability, and a flat threshold leaves that on the
table.

This is NOT the same hypothesis as S2<->S3 corroboration, which was just measured and closed at
+0.00001. Corroboration asked "does this candidate resemble my other match?". This asks "how many
matches should I expect at all?".

Counted directly from the full training ground truth - 2,206,821 entities, 7,638,365 links.
Source membership is read from the id PREFIX, so source2.tsv never has to be loaded.
"""
import collections
import sys

import pandas as pd

sys.path.insert(0, "src")
import config as C

p = C.PATHS["train"]
gt = pd.read_csv(p["ground_truth"], sep="\t", dtype=str).fillna("")
qcol, tcol = gt.columns[0], gt.columns[1]
print(f"ground truth rows: {len(gt):,}")

s1 = pd.read_parquet(p["source1"], columns=["entity_id", "country_norm"])
ctry = dict(zip(s1["entity_id"], s1["country_norm"].astype(str)))
N = len(s1)
print(f"S1 entities: {N:,}")

tot = collections.Counter()
combo = collections.Counter()
by_ctry = collections.defaultdict(collections.Counter)
listed = set()

for q, v in zip(gt[qcol], gt[tcol]):
    ms = [x for x in str(v).split(",") if x]
    c2 = sum(1 for m in ms if m.startswith("S2-"))
    c3 = len(ms) - c2
    tot[len(ms)] += 1
    combo[(c2, c3)] += 1
    by_ctry[ctry.get(q, "?")][len(ms)] += 1
    listed.add(q)

# entities absent from the ground truth are genuine singletons
n_missing = N - len(listed)
tot[0] += n_missing
combo[(0, 0)] += n_missing
for e, c in ctry.items():
    if e not in listed:
        by_ctry[c][0] += 1

print("\n=== total true matches per S1 entity ===")
cum = 0
for k in sorted(tot):
    cum += tot[k]
    print(f"  T = {k:<3} {tot[k]:>9,}  ({tot[k]/N:7.3%})  cum {cum/N:7.3%}")
print(f"  mean T = {sum(k*v for k, v in tot.items())/N:.4f}")

print("\n=== joint (from S2, from S3) - top 15 ===")
for (c2, c3), v in combo.most_common(15):
    print(f"  ({c2} from S2, {c3} from S3): {v:>9,}  ({v/N:7.3%})")

print("\n=== marginals ===")
m2 = collections.Counter()
m3 = collections.Counter()
for (c2, c3), v in combo.items():
    m2[c2] += v
    m3[c3] += v
print("  from S2:", {k: f"{v/N:.2%}" for k, v in sorted(m2.items())[:7]})
print("  from S3:", {k: f"{v/N:.2%}" for k, v in sorted(m3.items())[:7]})

print("\n=== by country ===")
for c in sorted(by_ctry):
    d = by_ctry[c]
    n = sum(d.values())
    if n < 1000:
        continue
    top = ", ".join(f"T={k}:{v/n:.1%}" for k, v in sorted(d.items())[:6])
    print(f"  {c:<8} n={n:>9,}  mean {sum(k*v for k, v in d.items())/n:.3f}   {top}")

print("\n=== HOW PREDICTABLE IS T? ===")
singl = tot.get(0, 0)
nonsing = N - singl
print(f"  singletons (T=0)       : {singl:,}  ({singl/N:.3%})")
print(f"  non-singletons         : {nonsing:,}")
rank = sorted(((v, k) for k, v in tot.items() if k > 0), reverse=True)
print(f"  most common non-zero T : T={rank[0][1]} at {rank[0][0]:,} "
      f"({rank[0][0]/nonsing:.2%} of non-singletons)")
print(f"  top-3 non-zero T cover : {sum(v for v, _ in rank[:3])/nonsing:.2%} "
      f"(values {[k for _, k in rank[:3]]})")

# The decisive question: does knowing the S2 count pin down the S3 count? If the conditional
# distribution of c3 given c2 is sharp, a structural prior is worth something; if it is as broad
# as the marginal, it is not.
print("\n=== IS c3 PREDICTABLE FROM c2?  (sharp conditional => exploitable) ===")
print(f"{'c2':>4}{'n':>11}{'mode c3':>9}{'P(mode)':>9}{'mean c3':>9}")
for c2 in sorted(m2)[:6]:
    sub = {c3: v for (a, c3), v in combo.items() if a == c2}
    n = sum(sub.values())
    if n < 1000:
        continue
    mode = max(sub.items(), key=lambda x: x[1])
    mean = sum(k * v for k, v in sub.items()) / n
    print(f"{c2:>4}{n:>11,}{mode[0]:>9}{mode[1]/n:>9.2%}{mean:>9.3f}")
marg_mode = max(m3.items(), key=lambda x: x[1])
print(f"  marginal mode of c3: {marg_mode[0]} at {marg_mode[1]/N:.2%}")
print("  If P(mode c3 | c2) is not much sharper than the marginal, a cardinality prior adds")
print("  nothing beyond what the probabilities already say.")