| """Does the France text pipeline actually degrade? Label-free, measured on test data. |
| |
| Three questions: |
| 1. How prevalent are French legal forms (sa/sarl/sas/sasu/sci/eurl/snc/scop/scm/selarl) |
| in French test records, and does `legal_suffix` capture them (it was built from a |
| US/India-derived list)? |
| 2. If a suffix is NOT stripped into legal_suffix, it stays in name_core. Then two |
| records for the same business that differ only in legal form look like a name |
| mismatch to every string feature. |
| 3. Same check for the US/India forms, as the control. |
| """ |
| import sys, collections, re |
| import pandas as pd |
| sys.path.insert(0, "src") |
| import config as C |
|
|
| FR = ["sa","sarl","sas","sasu","sci","eurl","snc","scop","scm","selarl","sel","scs","gie","scic","sca","ei","eirl"] |
| US = ["inc","llc","ltd","corp","co","lp","llp","pllc","pc","plc","pvt","limited","private"] |
|
|
| p = C.PATHS["test"] |
| cols = ["entity_id","business_name","country","country_norm","name_norm","name_core","legal_suffix"] |
| frames = [] |
| for k in ("source1","source2","source3"): |
| d = pd.read_parquet(p[k], columns=cols) |
| d["src"] = k |
| frames.append(d) |
| t = pd.concat(frames, ignore_index=True) |
| print(f"test records: {len(t):,}") |
| print("countries:", t["country_norm"].value_counts().to_dict()) |
|
|
| def probe(sub, label, forms): |
| n = len(sub) |
| core_tok = sub["name_core"].str.lower().str.split() |
| print(f"\n--- {label} n={n:,} ---") |
| print(f" legal_suffix non-empty: {(sub['legal_suffix'].astype(str).str.len()>0).mean():.4f}") |
| got = collections.Counter() |
| for f in forms: |
| |
| m = core_tok.apply(lambda ts, f=f: f in ts if isinstance(ts, list) else False) |
| if m.sum(): |
| got[f] = int(m.sum()) |
| tot = 0 |
| for f, c in got.most_common(20): |
| print(f" '{f}' still inside name_core: {c:>8,} ({c/n:.3%})") |
| tot += c |
| print(f" TOTAL records with an unstripped legal form in name_core: ~{tot:,} ({tot/n:.2%})") |
| return tot / n |
|
|
| for c in sorted(t["country_norm"].unique()): |
| sub = t[t["country_norm"] == c] |
| forms = FR if "fr" in str(c).lower() else US |
| probe(sub, f"{c} (probing {'FR' if forms is FR else 'US/IN'} forms)", forms) |
|
|
| |
| print("\n=== cross-check: FR forms inside non-France records, US forms inside France ===") |
| for c in sorted(t["country_norm"].unique()): |
| sub = t[t["country_norm"] == c] |
| other = US if "fr" in str(c).lower() else FR |
| probe(sub, f"{c} (probing the OTHER family)", other) |
|
|
