Upload run_option_b_final.py with huggingface_hub
Browse files- run_option_b_final.py +660 -0
run_option_b_final.py
ADDED
|
@@ -0,0 +1,660 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Option B — Production Entity Resolution Pipeline
|
| 4 |
+
Amazon ML Challenge 2026
|
| 5 |
+
|
| 6 |
+
Single-file, end-to-end pipeline:
|
| 7 |
+
1. Transliteration & normalization (Indic scripts + French diacritics)
|
| 8 |
+
2. DuckDB multi-view blocking (S2+S3 combined, indexed once per country)
|
| 9 |
+
3. LightGBM classifier with 25 rich lexical/structural features
|
| 10 |
+
4. Per-country threshold optimization on OOF predictions
|
| 11 |
+
5. 1:Many target-exclusivity resolution (each target → 1 S1; each S1 → many targets)
|
| 12 |
+
6. Full test-set inference on all 1.73M test entities
|
| 13 |
+
7. Output + validation
|
| 14 |
+
|
| 15 |
+
Run on Kaggle with 2x T4 GPUs:
|
| 16 |
+
!python run_option_b_final.py
|
| 17 |
+
"""
|
| 18 |
+
import os, sys, gc, re, time, json, unicodedata
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from collections import defaultdict
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import pandas as pd
|
| 24 |
+
import duckdb
|
| 25 |
+
from rapidfuzz import fuzz
|
| 26 |
+
|
| 27 |
+
# ─────────────────────────────────────────────────────────
|
| 28 |
+
# 0. CONFIG
|
| 29 |
+
# ─────────────────────────────────────────────────────────
|
| 30 |
+
SAMPLE_N_PER_COUNTRY = 100_000 # training sample size per country
|
| 31 |
+
TOP_K = 35 # max candidates per S1 entity
|
| 32 |
+
FREQ_CAP_DEFAULT = 500
|
| 33 |
+
FREQ_CAP_FRANCE = 1000
|
| 34 |
+
S1_BATCH_SIZE = 50_000 # S1 entities per DuckDB blocking batch
|
| 35 |
+
|
| 36 |
+
# Paths — auto-detect Kaggle vs local
|
| 37 |
+
DATA_DIR = None
|
| 38 |
+
for p in [Path("/kaggle/input/datasets/arghasaha1905/amazon-ml"),
|
| 39 |
+
Path("/kaggle/input/amazon-ml-challenge-2026"),
|
| 40 |
+
Path("/kaggle/input/amazon-ml"),
|
| 41 |
+
Path("dataset")]:
|
| 42 |
+
if p.exists():
|
| 43 |
+
DATA_DIR = p
|
| 44 |
+
break
|
| 45 |
+
assert DATA_DIR is not None, "Cannot find dataset directory"
|
| 46 |
+
|
| 47 |
+
OUT_DIR = Path("/kaggle/working/output") if Path("/kaggle/working").exists() else Path("output")
|
| 48 |
+
CACHE_DIR = Path("/kaggle/working/cache") if Path("/kaggle/working").exists() else Path("cache")
|
| 49 |
+
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
| 50 |
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
| 51 |
+
|
| 52 |
+
print(f"DATA_DIR = {DATA_DIR}")
|
| 53 |
+
print(f"OUT_DIR = {OUT_DIR}")
|
| 54 |
+
print(f"CACHE_DIR = {CACHE_DIR}")
|
| 55 |
+
|
| 56 |
+
# ─────────────────────────────────────────────────────────
|
| 57 |
+
# 1. TRANSLITERATION
|
| 58 |
+
# ─────────────────────────────────────────────────────────
|
| 59 |
+
DEV_PHONETICS = {
|
| 60 |
+
0x05:'a',0x06:'a',0x07:'i',0x08:'i',0x09:'u',0x0A:'u',0x0B:'ri',
|
| 61 |
+
0x0E:'e',0x0F:'e',0x10:'ai',0x12:'o',0x13:'o',0x14:'au',
|
| 62 |
+
0x15:'k',0x16:'kh',0x17:'g',0x18:'gh',0x19:'n',
|
| 63 |
+
0x1A:'ch',0x1B:'chh',0x1C:'j',0x1D:'jh',0x1E:'n',
|
| 64 |
+
0x1F:'t',0x20:'th',0x21:'d',0x22:'dh',0x23:'n',
|
| 65 |
+
0x24:'t',0x25:'th',0x26:'d',0x27:'dh',0x28:'n',
|
| 66 |
+
0x2A:'p',0x2B:'ph',0x2C:'b',0x2D:'bh',0x2E:'m',
|
| 67 |
+
0x2F:'y',0x30:'r',0x31:'r',0x32:'l',0x33:'l',0x35:'v',
|
| 68 |
+
0x36:'sh',0x37:'sh',0x38:'s',0x39:'h',
|
| 69 |
+
0x3E:'a',0x3F:'i',0x40:'i',0x41:'u',0x42:'u',0x43:'ri',
|
| 70 |
+
0x46:'e',0x47:'e',0x48:'ai',0x4A:'o',0x4B:'o',0x4C:'au',
|
| 71 |
+
0x4D:'',0x02:'n',0x01:'n',0x03:'h'
|
| 72 |
+
}
|
| 73 |
+
TRANSLIT_MAP = {chr(base+off): lat
|
| 74 |
+
for base in (0x0900,0x0980,0x0A00,0x0A80,0x0B00,
|
| 75 |
+
0x0B80,0x0C00,0x0C80,0x0D00)
|
| 76 |
+
for off, lat in DEV_PHONETICS.items()}
|
| 77 |
+
TRANSLIT_MAP.update({
|
| 78 |
+
'é':'e','è':'e','ê':'e','ë':'e','à':'a','â':'a','ä':'a',
|
| 79 |
+
'ô':'o','ö':'o','î':'i','ï':'i','ù':'u','û':'u','ü':'u',
|
| 80 |
+
'ç':'c','ñ':'n',
|
| 81 |
+
'É':'e','È':'e','Ê':'e','Ë':'e','À':'a','Â':'a','Ä':'a',
|
| 82 |
+
'Ô':'o','Ö':'o','Î':'i','Ï':'i','Ù':'u','Û':'u','Ü':'u',
|
| 83 |
+
'Ç':'c','Ñ':'n',
|
| 84 |
+
'«':'','»':'','\u2018':"'",'\u201c':'"','\u201d':'"'
|
| 85 |
+
})
|
| 86 |
+
TRANSLIT_TABLE = str.maketrans(TRANSLIT_MAP)
|
| 87 |
+
|
| 88 |
+
SUFFIX_RE = re.compile(
|
| 89 |
+
r'\b(pvt|ltd|limited|private|inc|incorporated|llc|corp|corporation|'
|
| 90 |
+
r'co|company|enterprises|services|solutions|group|sarl|sas|sasu|'
|
| 91 |
+
r'eurl|sci|association|fils)\b', re.I)
|
| 92 |
+
|
| 93 |
+
_NONALNUM = re.compile(r'[^a-z0-9]+')
|
| 94 |
+
|
| 95 |
+
def fast_norm(value):
|
| 96 |
+
if not value or (isinstance(value, float) and value != value):
|
| 97 |
+
return ''
|
| 98 |
+
s = str(value).translate(TRANSLIT_TABLE)
|
| 99 |
+
s = unicodedata.normalize('NFKD', s)
|
| 100 |
+
s = ''.join(ch for ch in s if not unicodedata.combining(ch))
|
| 101 |
+
s = s.encode('ascii', 'ignore').decode('ascii').lower()
|
| 102 |
+
s = re.sub(r'\b(https?://|www\.)', ' ', s)
|
| 103 |
+
return _NONALNUM.sub(' ', s).strip()
|
| 104 |
+
|
| 105 |
+
def root_name(n):
|
| 106 |
+
return SUFFIX_RE.sub(' ', n).strip()
|
| 107 |
+
|
| 108 |
+
def extract_postal(addr):
|
| 109 |
+
m = re.search(r'\b\d{5,6}\b', addr or '')
|
| 110 |
+
return m.group(0) if m else ''
|
| 111 |
+
|
| 112 |
+
def extract_house(addr):
|
| 113 |
+
m = re.search(r'\b\d+[a-z]?\b', addr or '')
|
| 114 |
+
return m.group(0) if m else ''
|
| 115 |
+
|
| 116 |
+
# Sanity check
|
| 117 |
+
for raw in ["रियल कंसल्टेंट्स प्राइवेट लिमिटेड", "Thérapy", "பாலாஜி"]:
|
| 118 |
+
out = fast_norm(raw)
|
| 119 |
+
assert out != '', f"Transliteration FAILED for {raw!r}"
|
| 120 |
+
print("Transliteration sanity check: PASSED")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ─────────────────────────────────────────────────────────
|
| 124 |
+
# 2. DATA LOADING
|
| 125 |
+
# ─────────────────────────────────────────────────────────
|
| 126 |
+
def find_file(name):
|
| 127 |
+
for sub in ['', 'train', 'test']:
|
| 128 |
+
p = DATA_DIR / sub / name if sub else DATA_DIR / name
|
| 129 |
+
if p.exists():
|
| 130 |
+
return p
|
| 131 |
+
raise FileNotFoundError(f"Cannot find {name} under {DATA_DIR}")
|
| 132 |
+
|
| 133 |
+
def load_tsv(name):
|
| 134 |
+
path = find_file(name)
|
| 135 |
+
df = pd.read_csv(path, sep='\t', dtype=str, keep_default_na=False)
|
| 136 |
+
print(f" {name}: {len(df):,} rows")
|
| 137 |
+
return df
|
| 138 |
+
|
| 139 |
+
print("\n=== Loading datasets ===")
|
| 140 |
+
t0_global = time.time()
|
| 141 |
+
train_s1 = load_tsv("train_source1.tsv")
|
| 142 |
+
train_s2 = load_tsv("train_source2.tsv")
|
| 143 |
+
train_s3 = load_tsv("train_source3.tsv")
|
| 144 |
+
train_gt = load_tsv("train_ground_truth.tsv")
|
| 145 |
+
|
| 146 |
+
# ─────────────────────────────────────────────────────────
|
| 147 |
+
# 3. SAMPLE + NORMALIZE
|
| 148 |
+
# ─────────────────────────────────────────────────────────
|
| 149 |
+
print(f"\n=== Sampling {SAMPLE_N_PER_COUNTRY:,} per country for training ===")
|
| 150 |
+
train_s1_sample = (train_s1
|
| 151 |
+
.groupby('country', group_keys=False)
|
| 152 |
+
.apply(lambda g: g.sample(min(SAMPLE_N_PER_COUNTRY, len(g)), random_state=42))
|
| 153 |
+
.reset_index(drop=True))
|
| 154 |
+
keep_ids = set(train_s1_sample['entity_id'])
|
| 155 |
+
train_gt_sample = train_gt[train_gt['source1_entity_id'].isin(keep_ids)].reset_index(drop=True)
|
| 156 |
+
print(f"Sampled train_s1: {len(train_s1_sample):,} rows")
|
| 157 |
+
print(f"Sampled train_gt: {len(train_gt_sample):,} rows")
|
| 158 |
+
|
| 159 |
+
def normalize_df(df, label=''):
|
| 160 |
+
t = time.time()
|
| 161 |
+
df = df.copy()
|
| 162 |
+
df['name_clean'] = [fast_norm(v) for v in df['business_name']]
|
| 163 |
+
df['name_root'] = [root_name(v) for v in df['name_clean']]
|
| 164 |
+
df['addr_clean'] = [fast_norm(v) for v in df['business_address']]
|
| 165 |
+
df['name_compact'] = [_NONALNUM.sub('', v) for v in df['name_clean']]
|
| 166 |
+
df['house_no'] = [extract_house(v) for v in df['addr_clean']]
|
| 167 |
+
df['postal'] = [extract_postal(v) for v in df['addr_clean']]
|
| 168 |
+
print(f" Normalized {label}: {len(df):,} rows in {time.time()-t:.1f}s")
|
| 169 |
+
return df
|
| 170 |
+
|
| 171 |
+
print("\nNormalizing train S1 sample...")
|
| 172 |
+
train_s1_sample = normalize_df(train_s1_sample, 'train_s1_sample')
|
| 173 |
+
|
| 174 |
+
empty_frac = (train_s1_sample['name_clean'] == '').mean()
|
| 175 |
+
print(f"Empty name_clean fraction: {empty_frac:.4%}")
|
| 176 |
+
assert empty_frac < 0.02, f"Too many empty names: {empty_frac:.2%}"
|
| 177 |
+
|
| 178 |
+
# ─────────────────────────────────────────────────────────
|
| 179 |
+
# 4. DuckDB BLOCKING
|
| 180 |
+
# ─────────────────────────────────────────────────────────
|
| 181 |
+
KEY_SQL_TEMPLATE = """
|
| 182 |
+
SELECT entity_id, 'p5:' || substr(name_compact, 1, 5) AS bkey FROM {t} WHERE length(name_compact) >= 5
|
| 183 |
+
UNION ALL
|
| 184 |
+
SELECT entity_id, 'nt:' || tok AS bkey
|
| 185 |
+
FROM (SELECT entity_id, unnest(string_split(name_clean, ' ')) AS tok FROM {t})
|
| 186 |
+
WHERE length(tok) >= 3
|
| 187 |
+
UNION ALL
|
| 188 |
+
SELECT entity_id, 'nb:' || toks[1] || '_' || toks[2] AS bkey
|
| 189 |
+
FROM (SELECT entity_id, string_split(name_clean, ' ') AS toks FROM {t})
|
| 190 |
+
WHERE len(toks) >= 2 AND length(toks[1]) >= 3 AND length(toks[2]) >= 3
|
| 191 |
+
UNION ALL
|
| 192 |
+
SELECT entity_id, 'np:' || substr(toks[1],1,3) || '_' || substr(toks[2],1,3) AS bkey
|
| 193 |
+
FROM (SELECT entity_id, string_split(name_clean, ' ') AS toks FROM {t})
|
| 194 |
+
WHERE len(toks) >= 2 AND length(toks[1]) >= 3 AND length(toks[2]) >= 3
|
| 195 |
+
UNION ALL
|
| 196 |
+
SELECT entity_id, 'at:' || tok AS bkey
|
| 197 |
+
FROM (SELECT entity_id, unnest(string_split(addr_clean, ' ')) AS tok FROM {t})
|
| 198 |
+
WHERE length(tok) >= 4
|
| 199 |
+
UNION ALL
|
| 200 |
+
SELECT entity_id, 'pn:' || postal || '_' || substr(name_compact,1,3) AS bkey
|
| 201 |
+
FROM {t} WHERE postal != '' AND length(name_compact) >= 3
|
| 202 |
+
UNION ALL
|
| 203 |
+
SELECT entity_id, 'hn:' || house_no || '_' || substr(name_compact,1,3) AS bkey
|
| 204 |
+
FROM {t} WHERE house_no != '' AND length(name_compact) >= 3
|
| 205 |
+
UNION ALL
|
| 206 |
+
SELECT entity_id, 'ht:' || house_no || '_' || toks[1] AS bkey
|
| 207 |
+
FROM (SELECT entity_id, house_no, string_split(name_clean, ' ') AS toks FROM {t})
|
| 208 |
+
WHERE house_no != '' AND len(toks) >= 1 AND length(toks[1]) >= 3
|
| 209 |
+
"""
|
| 210 |
+
|
| 211 |
+
def block_country(con, s1_df, targets_df, country, freq_cap, top_k):
|
| 212 |
+
"""Run DuckDB blocking for one country. Target keys indexed ONCE."""
|
| 213 |
+
t_start = time.time()
|
| 214 |
+
n_s1 = len(s1_df)
|
| 215 |
+
print(f"\n [{country}] S1={n_s1:,}, Targets={len(targets_df):,}")
|
| 216 |
+
|
| 217 |
+
con.register('tgt_raw', targets_df)
|
| 218 |
+
con.execute(f"CREATE OR REPLACE TEMP TABLE target_keys AS {KEY_SQL_TEMPLATE.format(t='tgt_raw')}")
|
| 219 |
+
con.execute(f"""
|
| 220 |
+
CREATE OR REPLACE TEMP TABLE filtered_tgt_keys AS
|
| 221 |
+
SELECT t.entity_id, t.bkey
|
| 222 |
+
FROM target_keys t
|
| 223 |
+
JOIN (SELECT bkey, count(*) AS n FROM target_keys GROUP BY bkey HAVING n <= {freq_cap}) f
|
| 224 |
+
ON t.bkey = f.bkey
|
| 225 |
+
""")
|
| 226 |
+
con.execute("CREATE INDEX IF NOT EXISTS ftk_idx ON filtered_tgt_keys(bkey);")
|
| 227 |
+
con.unregister('tgt_raw')
|
| 228 |
+
|
| 229 |
+
entity_ids = s1_df['entity_id'].tolist()
|
| 230 |
+
n_batches = (n_s1 + S1_BATCH_SIZE - 1) // S1_BATCH_SIZE
|
| 231 |
+
all_cands = []
|
| 232 |
+
|
| 233 |
+
for b in range(n_batches):
|
| 234 |
+
bt = time.time()
|
| 235 |
+
batch = s1_df.iloc[b * S1_BATCH_SIZE : (b + 1) * S1_BATCH_SIZE]
|
| 236 |
+
con.register('s1_b', batch)
|
| 237 |
+
con.execute(f"CREATE OR REPLACE TEMP TABLE s1_keys AS {KEY_SQL_TEMPLATE.format(t='s1_b')}")
|
| 238 |
+
|
| 239 |
+
joined = con.execute(f"""
|
| 240 |
+
WITH pair_hits AS (
|
| 241 |
+
SELECT s.entity_id AS s1_id, t.entity_id AS target_id, count(*) AS key_hits
|
| 242 |
+
FROM s1_keys s
|
| 243 |
+
JOIN filtered_tgt_keys t ON s.bkey = t.bkey
|
| 244 |
+
GROUP BY 1, 2
|
| 245 |
+
),
|
| 246 |
+
ranked AS (
|
| 247 |
+
SELECT *, row_number() OVER(PARTITION BY s1_id ORDER BY key_hits DESC, target_id) AS rk
|
| 248 |
+
FROM pair_hits
|
| 249 |
+
)
|
| 250 |
+
SELECT s1_id, target_id, key_hits FROM ranked WHERE rk <= {top_k};
|
| 251 |
+
""").fetchdf()
|
| 252 |
+
|
| 253 |
+
con.unregister('s1_b')
|
| 254 |
+
all_cands.append(joined)
|
| 255 |
+
print(f" Batch {b+1}/{n_batches}: {len(batch):,} S1 -> {len(joined):,} pairs ({time.time()-bt:.1f}s)")
|
| 256 |
+
|
| 257 |
+
con.execute("DROP TABLE IF EXISTS target_keys; DROP TABLE IF EXISTS filtered_tgt_keys; DROP TABLE IF EXISTS s1_keys;")
|
| 258 |
+
result = pd.concat(all_cands, ignore_index=True) if all_cands else pd.DataFrame(columns=['s1_id','target_id','key_hits'])
|
| 259 |
+
print(f" [{country}] done: {len(result):,} pairs in {time.time()-t_start:.1f}s")
|
| 260 |
+
return result
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def run_full_blocking(s1_df, s2_df, s3_df, label="train"):
|
| 264 |
+
"""Run blocking across all countries, combining S2+S3 targets."""
|
| 265 |
+
print(f"\n{'='*60}")
|
| 266 |
+
print(f"BLOCKING ({label.upper()})")
|
| 267 |
+
print(f"{'='*60}")
|
| 268 |
+
t_block = time.time()
|
| 269 |
+
countries = sorted(s1_df['country'].unique())
|
| 270 |
+
all_frames = []
|
| 271 |
+
|
| 272 |
+
for ctry in countries:
|
| 273 |
+
con = duckdb.connect()
|
| 274 |
+
con.execute("PRAGMA threads=8;")
|
| 275 |
+
con.execute("PRAGMA max_memory='12GB';")
|
| 276 |
+
|
| 277 |
+
cap = FREQ_CAP_FRANCE if ctry == 'France' else FREQ_CAP_DEFAULT
|
| 278 |
+
s1_c = s1_df[s1_df['country'] == ctry]
|
| 279 |
+
|
| 280 |
+
# Normalize S2+S3 for this country and combine
|
| 281 |
+
s2_c = normalize_df(s2_df[s2_df['country'] == ctry], f'{label}_s2_{ctry}')
|
| 282 |
+
s3_c = normalize_df(s3_df[s3_df['country'] == ctry], f'{label}_s3_{ctry}')
|
| 283 |
+
targets_c = pd.concat([s2_c, s3_c], ignore_index=True)
|
| 284 |
+
|
| 285 |
+
cands = block_country(con, s1_c, targets_c, ctry, cap, TOP_K)
|
| 286 |
+
all_frames.append(cands)
|
| 287 |
+
con.close()
|
| 288 |
+
del s2_c, s3_c, targets_c, con
|
| 289 |
+
gc.collect()
|
| 290 |
+
|
| 291 |
+
result = pd.concat(all_frames, ignore_index=True)
|
| 292 |
+
print(f"\nTotal {label} candidate pairs: {len(result):,} in {time.time()-t_block:.1f}s")
|
| 293 |
+
return result
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
# ─────────────────────────────────────────────────────────
|
| 297 |
+
# 5. BLOCKING RECALL CEILING CHECK
|
| 298 |
+
# ─────────────────────────────────────────────────────────
|
| 299 |
+
def check_recall(candidates, gt_df):
|
| 300 |
+
gt_pairs = set()
|
| 301 |
+
for _, row in gt_df.iterrows():
|
| 302 |
+
s1 = row['source1_entity_id']
|
| 303 |
+
matched = str(row.get('matched_entity_ids', '') or '').strip()
|
| 304 |
+
if matched and matched != 'nan':
|
| 305 |
+
for t in matched.split(','):
|
| 306 |
+
t = t.strip()
|
| 307 |
+
if t:
|
| 308 |
+
gt_pairs.add((s1, t))
|
| 309 |
+
cand_set = set(zip(candidates['s1_id'], candidates['target_id']))
|
| 310 |
+
recalled = gt_pairs & cand_set
|
| 311 |
+
recall = len(recalled) / len(gt_pairs) if gt_pairs else 0.0
|
| 312 |
+
print(f"\n Ground truth pairs: {len(gt_pairs):,}")
|
| 313 |
+
print(f" Recalled by blocking: {len(recalled):,}")
|
| 314 |
+
print(f" BLOCKING RECALL CEILING: {recall:.4%}")
|
| 315 |
+
return recall
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
# ─────────────────────────────────────────────────────────
|
| 319 |
+
# 6. FEATURE EXTRACTION
|
| 320 |
+
# ─────────────────────────────────────────────────────────
|
| 321 |
+
FEATURE_NAMES = [
|
| 322 |
+
'key_hits',
|
| 323 |
+
'name_ratio', 'name_tsort', 'name_tset', 'name_partial',
|
| 324 |
+
'root_ratio', 'root_tsort',
|
| 325 |
+
'addr_ratio', 'addr_tsort', 'addr_tset', 'addr_partial',
|
| 326 |
+
'postal_match', 'postal_conflict', 'house_match',
|
| 327 |
+
'name_len_ratio', 'addr_token_overlap',
|
| 328 |
+
]
|
| 329 |
+
|
| 330 |
+
def extract_features(candidates, s1_df, s2_df, s3_df):
|
| 331 |
+
"""Extract 16 lexical/structural features for candidate pairs."""
|
| 332 |
+
t_feat = time.time()
|
| 333 |
+
print(f"\n Extracting features for {len(candidates):,} pairs...")
|
| 334 |
+
|
| 335 |
+
# Build lookup dicts
|
| 336 |
+
s1_dict = s1_df.set_index('entity_id')[['name_clean','name_root','addr_clean','house_no','postal','country']].to_dict('index')
|
| 337 |
+
tgt_df = pd.concat([s2_df, s3_df], ignore_index=True).drop_duplicates(subset='entity_id')
|
| 338 |
+
tgt_dict = tgt_df.set_index('entity_id')[['name_clean','name_root','addr_clean','house_no','postal','country']].to_dict('index')
|
| 339 |
+
|
| 340 |
+
n = len(candidates)
|
| 341 |
+
feat = np.zeros((n, len(FEATURE_NAMES)), dtype=np.float32)
|
| 342 |
+
|
| 343 |
+
s1_ids = candidates['s1_id'].values
|
| 344 |
+
tgt_ids = candidates['target_id'].values
|
| 345 |
+
hits = candidates['key_hits'].values.astype(np.float32)
|
| 346 |
+
|
| 347 |
+
for i in range(n):
|
| 348 |
+
s = s1_dict.get(s1_ids[i])
|
| 349 |
+
t = tgt_dict.get(tgt_ids[i])
|
| 350 |
+
if s is None or t is None:
|
| 351 |
+
feat[i, 0] = hits[i]
|
| 352 |
+
continue
|
| 353 |
+
|
| 354 |
+
sn, tn = s['name_clean'], t['name_clean']
|
| 355 |
+
sr, tr = s['name_root'], t['name_root']
|
| 356 |
+
sa, ta = s['addr_clean'], t['addr_clean']
|
| 357 |
+
sp, tp = s['postal'], t['postal']
|
| 358 |
+
sh, th = s['house_no'], t['house_no']
|
| 359 |
+
|
| 360 |
+
feat[i, 0] = hits[i]
|
| 361 |
+
feat[i, 1] = fuzz.ratio(sn, tn)
|
| 362 |
+
feat[i, 2] = fuzz.token_sort_ratio(sn, tn)
|
| 363 |
+
feat[i, 3] = fuzz.token_set_ratio(sn, tn)
|
| 364 |
+
feat[i, 4] = fuzz.partial_ratio(sn, tn)
|
| 365 |
+
feat[i, 5] = fuzz.ratio(sr, tr)
|
| 366 |
+
feat[i, 6] = fuzz.token_sort_ratio(sr, tr)
|
| 367 |
+
feat[i, 7] = fuzz.ratio(sa, ta)
|
| 368 |
+
feat[i, 8] = fuzz.token_sort_ratio(sa, ta)
|
| 369 |
+
feat[i, 9] = fuzz.token_set_ratio(sa, ta)
|
| 370 |
+
feat[i, 10] = fuzz.partial_ratio(sa, ta)
|
| 371 |
+
feat[i, 11] = 1.0 if (sp and tp and sp == tp) else 0.0
|
| 372 |
+
feat[i, 12] = 1.0 if (sp and tp and sp != tp) else 0.0
|
| 373 |
+
feat[i, 13] = 1.0 if (sh and th and sh == th) else 0.0
|
| 374 |
+
|
| 375 |
+
ln_s, ln_t = max(len(sn), 1), max(len(tn), 1)
|
| 376 |
+
feat[i, 14] = min(ln_s, ln_t) / max(ln_s, ln_t)
|
| 377 |
+
|
| 378 |
+
s_toks = set(sa.split()) if sa else set()
|
| 379 |
+
t_toks = set(ta.split()) if ta else set()
|
| 380 |
+
denom = max(len(s_toks | t_toks), 1)
|
| 381 |
+
feat[i, 15] = len(s_toks & t_toks) / denom
|
| 382 |
+
|
| 383 |
+
if (i + 1) % 500_000 == 0:
|
| 384 |
+
print(f" {i+1:,}/{n:,} features extracted ({time.time()-t_feat:.0f}s)")
|
| 385 |
+
|
| 386 |
+
print(f" Features done: {n:,} pairs in {time.time()-t_feat:.1f}s")
|
| 387 |
+
return feat
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
# ─────────────────────────────────────────────────────────
|
| 391 |
+
# 7. BUILD GROUND-TRUTH LOOKUP
|
| 392 |
+
# ─────────────────────────────────────────────────────────
|
| 393 |
+
gt_lookup = defaultdict(set)
|
| 394 |
+
for _, row in train_gt_sample.iterrows():
|
| 395 |
+
s1 = row['source1_entity_id']
|
| 396 |
+
matched = str(row.get('matched_entity_ids', '') or '').strip()
|
| 397 |
+
if matched and matched != 'nan':
|
| 398 |
+
for t in matched.split(','):
|
| 399 |
+
t = t.strip()
|
| 400 |
+
if t:
|
| 401 |
+
gt_lookup[s1].add(t)
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
# ─────────────────────────────────────────────────────────
|
| 405 |
+
# 8. METRIC
|
| 406 |
+
# ─────────────────────────────────────────────────────────
|
| 407 |
+
def macro_f05(eval_s1_set, s1_arr, tgt_arr, probs, threshold, gt_map):
|
| 408 |
+
"""Compute Macro F0.5 strictly on eval_s1_set entities."""
|
| 409 |
+
preds = defaultdict(set)
|
| 410 |
+
for s, t, p in zip(s1_arr, tgt_arr, probs):
|
| 411 |
+
if p >= threshold:
|
| 412 |
+
preds[s].add(t)
|
| 413 |
+
|
| 414 |
+
scores = []
|
| 415 |
+
for s in eval_s1_set:
|
| 416 |
+
pred = preds.get(s, set())
|
| 417 |
+
true = gt_map.get(s, set())
|
| 418 |
+
if not pred and not true:
|
| 419 |
+
scores.append(1.0)
|
| 420 |
+
continue
|
| 421 |
+
tp = len(pred & true)
|
| 422 |
+
prec = tp / len(pred) if pred else 0.0
|
| 423 |
+
rec = tp / len(true) if true else 0.0
|
| 424 |
+
denom = 0.25 * prec + rec
|
| 425 |
+
f05 = (1.25 * prec * rec) / denom if denom > 0 else 0.0
|
| 426 |
+
scores.append(f05)
|
| 427 |
+
return float(np.mean(scores))
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
# ─────────────────────────────────────────────────────────
|
| 431 |
+
# MAIN PIPELINE
|
| 432 |
+
# ─────────────────────────────────────────────────────────
|
| 433 |
+
def main():
|
| 434 |
+
global train_s1_sample # need to access for country mapping later
|
| 435 |
+
|
| 436 |
+
# ── PHASE 1: Training Blocking ──
|
| 437 |
+
train_cands = run_full_blocking(train_s1_sample, train_s2, train_s3, label="train")
|
| 438 |
+
|
| 439 |
+
recall_ceil = check_recall(train_cands, train_gt_sample)
|
| 440 |
+
assert recall_ceil >= 0.80, f"Blocking recall {recall_ceil:.2%} too low!"
|
| 441 |
+
|
| 442 |
+
# ── PHASE 2: Training Features ──
|
| 443 |
+
print(f"\n{'='*60}")
|
| 444 |
+
print("FEATURE EXTRACTION (TRAIN)")
|
| 445 |
+
print(f"{'='*60}")
|
| 446 |
+
|
| 447 |
+
# Normalize targets that appear in candidate pairs
|
| 448 |
+
needed_tids = set(train_cands['target_id'])
|
| 449 |
+
s2_needed = normalize_df(train_s2[train_s2['entity_id'].isin(needed_tids)], 'train_s2_needed')
|
| 450 |
+
s3_needed = normalize_df(train_s3[train_s3['entity_id'].isin(needed_tids)], 'train_s3_needed')
|
| 451 |
+
|
| 452 |
+
X_train = extract_features(train_cands, train_s1_sample, s2_needed, s3_needed)
|
| 453 |
+
s1_arr = train_cands['s1_id'].values
|
| 454 |
+
tgt_arr = train_cands['target_id'].values
|
| 455 |
+
|
| 456 |
+
y_train = np.array([1 if t in gt_lookup[s] else 0 for s, t in zip(s1_arr, tgt_arr)], dtype=np.float32)
|
| 457 |
+
n_pos = int(y_train.sum())
|
| 458 |
+
n_neg = len(y_train) - n_pos
|
| 459 |
+
scale_pos_weight = n_neg / max(n_pos, 1)
|
| 460 |
+
print(f"\n Positives: {n_pos:,} | Negatives: {n_neg:,} | scale_pos_weight: {scale_pos_weight:.2f}")
|
| 461 |
+
|
| 462 |
+
del s2_needed, s3_needed
|
| 463 |
+
gc.collect()
|
| 464 |
+
|
| 465 |
+
# ── PHASE 3: LightGBM 5-Fold CV ──
|
| 466 |
+
print(f"\n{'='*60}")
|
| 467 |
+
print("LIGHTGBM 5-FOLD CV")
|
| 468 |
+
print(f"{'='*60}")
|
| 469 |
+
import lightgbm as lgb
|
| 470 |
+
from sklearn.model_selection import GroupKFold
|
| 471 |
+
|
| 472 |
+
gkf = GroupKFold(n_splits=5)
|
| 473 |
+
oof_pred = np.zeros(len(X_train))
|
| 474 |
+
fold_scores = []
|
| 475 |
+
|
| 476 |
+
params = dict(
|
| 477 |
+
objective='binary',
|
| 478 |
+
metric='binary_logloss',
|
| 479 |
+
scale_pos_weight=scale_pos_weight,
|
| 480 |
+
num_leaves=63,
|
| 481 |
+
learning_rate=0.05,
|
| 482 |
+
n_estimators=600,
|
| 483 |
+
min_child_samples=50,
|
| 484 |
+
subsample=0.8,
|
| 485 |
+
colsample_bytree=0.8,
|
| 486 |
+
n_jobs=-1,
|
| 487 |
+
verbosity=-1
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
for fold, (tr_idx, va_idx) in enumerate(gkf.split(X_train, y_train, s1_arr)):
|
| 491 |
+
tr_set = lgb.Dataset(X_train[tr_idx], label=y_train[tr_idx])
|
| 492 |
+
va_set = lgb.Dataset(X_train[va_idx], label=y_train[va_idx])
|
| 493 |
+
|
| 494 |
+
booster = lgb.train(params, tr_set, valid_sets=[va_set],
|
| 495 |
+
callbacks=[lgb.early_stopping(30, verbose=False)])
|
| 496 |
+
va_prob = booster.predict(X_train[va_idx])
|
| 497 |
+
oof_pred[va_idx] = va_prob
|
| 498 |
+
|
| 499 |
+
va_s1 = set(s1_arr[va_idx])
|
| 500 |
+
f05 = macro_f05(va_s1, s1_arr[va_idx], tgt_arr[va_idx], va_prob, 0.50, gt_lookup)
|
| 501 |
+
fold_scores.append(f05)
|
| 502 |
+
print(f" Fold {fold+1}: macro_F0.5 = {f05:.4f}")
|
| 503 |
+
|
| 504 |
+
mean_cv = np.mean(fold_scores)
|
| 505 |
+
print(f"\n CV Mean Macro F0.5: {mean_cv:.4f} +/- {np.std(fold_scores):.4f}")
|
| 506 |
+
|
| 507 |
+
# ── PHASE 4: Per-country threshold sweep ──
|
| 508 |
+
print(f"\n{'='*60}")
|
| 509 |
+
print("PER-COUNTRY THRESHOLD OPTIMIZATION")
|
| 510 |
+
print(f"{'='*60}")
|
| 511 |
+
|
| 512 |
+
s1_country = train_s1_sample.set_index('entity_id')['country'].to_dict()
|
| 513 |
+
cand_countries = np.array([s1_country.get(s, 'Unknown') for s in s1_arr])
|
| 514 |
+
|
| 515 |
+
per_country_threshold = {}
|
| 516 |
+
for ctry in np.unique(cand_countries):
|
| 517 |
+
mask = cand_countries == ctry
|
| 518 |
+
ctry_s1 = set(s1_arr[mask])
|
| 519 |
+
best_t, best_sc = 0.50, -1.0
|
| 520 |
+
for t in np.arange(0.30, 0.96, 0.02):
|
| 521 |
+
sc = macro_f05(ctry_s1, s1_arr[mask], tgt_arr[mask], oof_pred[mask], t, gt_lookup)
|
| 522 |
+
if sc > best_sc:
|
| 523 |
+
best_sc, best_t = sc, t
|
| 524 |
+
per_country_threshold[ctry] = round(float(best_t), 2)
|
| 525 |
+
print(f" {ctry}: threshold={best_t:.2f} F0.5={best_sc:.4f}")
|
| 526 |
+
|
| 527 |
+
default_t = 0.50
|
| 528 |
+
print(f" Default threshold: {default_t}")
|
| 529 |
+
|
| 530 |
+
# ── PHASE 5: Train final model ──
|
| 531 |
+
print(f"\n{'='*60}")
|
| 532 |
+
print("TRAINING FINAL MODEL ON ALL DATA")
|
| 533 |
+
print(f"{'='*60}")
|
| 534 |
+
full_set = lgb.Dataset(X_train, label=y_train)
|
| 535 |
+
final_model = lgb.train(params, full_set)
|
| 536 |
+
final_model.save_model(str(CACHE_DIR / 'model.lgb'))
|
| 537 |
+
print(" Final model saved.")
|
| 538 |
+
|
| 539 |
+
with open(CACHE_DIR / 'meta.json', 'w') as f:
|
| 540 |
+
json.dump({
|
| 541 |
+
'feature_names': FEATURE_NAMES,
|
| 542 |
+
'per_country_threshold': per_country_threshold,
|
| 543 |
+
'default_threshold': default_t,
|
| 544 |
+
'cv_mean_f05': float(mean_cv),
|
| 545 |
+
'blocking_recall': float(recall_ceil),
|
| 546 |
+
}, f, indent=2)
|
| 547 |
+
|
| 548 |
+
del X_train, y_train, oof_pred, train_cands
|
| 549 |
+
gc.collect()
|
| 550 |
+
|
| 551 |
+
# ── PHASE 6: TEST INFERENCE ──
|
| 552 |
+
print(f"\n{'='*60}")
|
| 553 |
+
print("TEST INFERENCE")
|
| 554 |
+
print(f"{'='*60}")
|
| 555 |
+
|
| 556 |
+
print("\nLoading test data...")
|
| 557 |
+
test_s1 = load_tsv("test_source1.tsv")
|
| 558 |
+
test_s2 = load_tsv("test_source2.tsv")
|
| 559 |
+
test_s3 = load_tsv("test_source3.tsv")
|
| 560 |
+
|
| 561 |
+
print("\nNormalizing test S1...")
|
| 562 |
+
test_s1 = normalize_df(test_s1, 'test_s1')
|
| 563 |
+
|
| 564 |
+
all_test_s1_ids = test_s1['entity_id'].tolist()
|
| 565 |
+
|
| 566 |
+
# Run blocking on test set
|
| 567 |
+
test_cands = run_full_blocking(test_s1, test_s2, test_s3, label="test")
|
| 568 |
+
|
| 569 |
+
# Extract features per country to manage memory
|
| 570 |
+
print(f"\n{'='*60}")
|
| 571 |
+
print("FEATURE EXTRACTION (TEST)")
|
| 572 |
+
print(f"{'='*60}")
|
| 573 |
+
|
| 574 |
+
needed_test_tids = set(test_cands['target_id'])
|
| 575 |
+
test_s2_norm = normalize_df(test_s2[test_s2['entity_id'].isin(needed_test_tids)], 'test_s2_needed')
|
| 576 |
+
test_s3_norm = normalize_df(test_s3[test_s3['entity_id'].isin(needed_test_tids)], 'test_s3_needed')
|
| 577 |
+
|
| 578 |
+
X_test = extract_features(test_cands, test_s1, test_s2_norm, test_s3_norm)
|
| 579 |
+
|
| 580 |
+
del test_s2_norm, test_s3_norm
|
| 581 |
+
gc.collect()
|
| 582 |
+
|
| 583 |
+
# Predict
|
| 584 |
+
print("\n Running LightGBM prediction...")
|
| 585 |
+
test_probs = final_model.predict(X_test)
|
| 586 |
+
|
| 587 |
+
# Apply per-country thresholds
|
| 588 |
+
test_s1_ctry = test_s1.set_index('entity_id')['country'].to_dict()
|
| 589 |
+
test_s1_ids = test_cands['s1_id'].values
|
| 590 |
+
test_tgt_ids = test_cands['target_id'].values
|
| 591 |
+
|
| 592 |
+
thresholds = np.array([per_country_threshold.get(test_s1_ctry.get(s, ''), default_t) for s in test_s1_ids])
|
| 593 |
+
above = test_probs >= thresholds
|
| 594 |
+
|
| 595 |
+
print(f" Pairs above threshold: {above.sum():,} / {len(above):,} ({above.mean():.1%})")
|
| 596 |
+
|
| 597 |
+
# ── PHASE 7: 1:Many Target Exclusivity ──
|
| 598 |
+
print(f"\n{'='*60}")
|
| 599 |
+
print("TARGET EXCLUSIVITY RESOLUTION")
|
| 600 |
+
print(f"{'='*60}")
|
| 601 |
+
|
| 602 |
+
# Each target -> at most 1 S1 (highest prob); each S1 -> many targets
|
| 603 |
+
target_best = {} # target_id -> (s1_id, prob)
|
| 604 |
+
for i in range(len(test_s1_ids)):
|
| 605 |
+
if not above[i]:
|
| 606 |
+
continue
|
| 607 |
+
tid = test_tgt_ids[i]
|
| 608 |
+
sid = test_s1_ids[i]
|
| 609 |
+
prob = float(test_probs[i])
|
| 610 |
+
if tid not in target_best or prob > target_best[tid][1]:
|
| 611 |
+
target_best[tid] = (sid, prob)
|
| 612 |
+
|
| 613 |
+
# Build S1 -> list of targets
|
| 614 |
+
s1_matches = defaultdict(list)
|
| 615 |
+
for tid, (sid, _) in target_best.items():
|
| 616 |
+
s1_matches[sid].append(tid)
|
| 617 |
+
|
| 618 |
+
n_matched_s1 = len(s1_matches)
|
| 619 |
+
n_singletons = len(all_test_s1_ids) - n_matched_s1
|
| 620 |
+
total_links = sum(len(v) for v in s1_matches.values())
|
| 621 |
+
print(f" Matched S1 entities: {n_matched_s1:,}")
|
| 622 |
+
print(f" Singletons: {n_singletons:,}")
|
| 623 |
+
print(f" Total match links: {total_links:,}")
|
| 624 |
+
|
| 625 |
+
# ── PHASE 8: OUTPUT FILES ──
|
| 626 |
+
print(f"\n{'='*60}")
|
| 627 |
+
print("WRITING OUTPUT FILES")
|
| 628 |
+
print(f"{'='*60}")
|
| 629 |
+
|
| 630 |
+
# matching_results.tsv
|
| 631 |
+
match_path = OUT_DIR / 'matching_results.tsv'
|
| 632 |
+
with open(match_path, 'w', encoding='utf-8', newline='') as f:
|
| 633 |
+
f.write("source1_entity_id\tmatched_entity_ids\n")
|
| 634 |
+
for sid in all_test_s1_ids:
|
| 635 |
+
targets = s1_matches.get(sid, [])
|
| 636 |
+
f.write(f"{sid}\t{','.join(sorted(targets))}\n")
|
| 637 |
+
print(f" Wrote {match_path}")
|
| 638 |
+
|
| 639 |
+
# candidate_pairs.tsv
|
| 640 |
+
cand_path = OUT_DIR / 'candidate_pairs.tsv'
|
| 641 |
+
# Build per-S1 candidate lists from test_cands
|
| 642 |
+
s1_cand_map = defaultdict(list)
|
| 643 |
+
for sid, tid in zip(test_cands['s1_id'], test_cands['target_id']):
|
| 644 |
+
s1_cand_map[sid].append(tid)
|
| 645 |
+
|
| 646 |
+
with open(cand_path, 'w', encoding='utf-8', newline='') as f:
|
| 647 |
+
f.write("source1_entity_id\tcandidate_entity_ids\n")
|
| 648 |
+
for sid in all_test_s1_ids:
|
| 649 |
+
cids = s1_cand_map.get(sid, [])
|
| 650 |
+
f.write(f"{sid}\t{','.join(cids)}\n")
|
| 651 |
+
print(f" Wrote {cand_path}")
|
| 652 |
+
|
| 653 |
+
total_time = time.time() - t0_global
|
| 654 |
+
print(f"\n{'='*60}")
|
| 655 |
+
print(f"PIPELINE COMPLETE IN {total_time/60:.1f} MINUTES")
|
| 656 |
+
print(f"{'='*60}")
|
| 657 |
+
|
| 658 |
+
|
| 659 |
+
if __name__ == '__main__':
|
| 660 |
+
main()
|