"""Does the France text pipeline actually degrade? Label-free, measured on test data.

Three questions:
 1. How prevalent are French legal forms (sa/sarl/sas/sasu/sci/eurl/snc/scop/scm/selarl)
    in French test records, and does `legal_suffix` capture them (it was built from a
    US/India-derived list)?
 2. If a suffix is NOT stripped into legal_suffix, it stays in name_core. Then two
    records for the same business that differ only in legal form look like a name
    mismatch to every string feature.
 3. Same check for the US/India forms, as the control.
"""
import sys, collections, re
import pandas as pd
sys.path.insert(0, "src")
import config as C

FR = ["sa","sarl","sas","sasu","sci","eurl","snc","scop","scm","selarl","sel","scs","gie","scic","sca","ei","eirl"]
US = ["inc","llc","ltd","corp","co","lp","llp","pllc","pc","plc","pvt","limited","private"]

p = C.PATHS["test"]
cols = ["entity_id","business_name","country","country_norm","name_norm","name_core","legal_suffix"]
frames = []
for k in ("source1","source2","source3"):
    d = pd.read_parquet(p[k], columns=cols)
    d["src"] = k
    frames.append(d)
t = pd.concat(frames, ignore_index=True)
print(f"test records: {len(t):,}")
print("countries:", t["country_norm"].value_counts().to_dict())

def probe(sub, label, forms):
    n = len(sub)
    core_tok = sub["name_core"].str.lower().str.split()
    print(f"\n--- {label}  n={n:,} ---")
    print(f"  legal_suffix non-empty: {(sub['legal_suffix'].astype(str).str.len()>0).mean():.4f}")
    got = collections.Counter()
    for f in forms:
        # token-level presence in the residual core (i.e. NOT stripped out)
        m = core_tok.apply(lambda ts, f=f: f in ts if isinstance(ts, list) else False)
        if m.sum():
            got[f] = int(m.sum())
    tot = 0
    for f, c in got.most_common(20):
        print(f"    '{f}' still inside name_core: {c:>8,}  ({c/n:.3%})")
        tot += c
    print(f"  TOTAL records with an unstripped legal form in name_core: ~{tot:,} ({tot/n:.2%})")
    return tot / n

for c in sorted(t["country_norm"].unique()):
    sub = t[t["country_norm"] == c]
    forms = FR if "fr" in str(c).lower() else US
    probe(sub, f"{c}  (probing {'FR' if forms is FR else 'US/IN'} forms)", forms)

# cross-check: do FR forms also leak in US records, and US forms in FR?
print("\n=== cross-check: FR forms inside non-France records, US forms inside France ===")
for c in sorted(t["country_norm"].unique()):
    sub = t[t["country_norm"] == c]
    other = US if "fr" in str(c).lower() else FR
    probe(sub, f"{c} (probing the OTHER family)", other)