Oddproblem commited on
Commit
7477000
·
verified ·
1 Parent(s): 23a5f1f

Upload run_option_b_final.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. run_option_b_final.py +660 -0
run_option_b_final.py ADDED
@@ -0,0 +1,660 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Option B — Production Entity Resolution Pipeline
4
+ Amazon ML Challenge 2026
5
+
6
+ Single-file, end-to-end pipeline:
7
+ 1. Transliteration & normalization (Indic scripts + French diacritics)
8
+ 2. DuckDB multi-view blocking (S2+S3 combined, indexed once per country)
9
+ 3. LightGBM classifier with 25 rich lexical/structural features
10
+ 4. Per-country threshold optimization on OOF predictions
11
+ 5. 1:Many target-exclusivity resolution (each target → 1 S1; each S1 → many targets)
12
+ 6. Full test-set inference on all 1.73M test entities
13
+ 7. Output + validation
14
+
15
+ Run on Kaggle with 2x T4 GPUs:
16
+ !python run_option_b_final.py
17
+ """
18
+ import os, sys, gc, re, time, json, unicodedata
19
+ from pathlib import Path
20
+ from collections import defaultdict
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+ import duckdb
25
+ from rapidfuzz import fuzz
26
+
27
+ # ─────────────────────────────────────────────────────────
28
+ # 0. CONFIG
29
+ # ─────────────────────────────────────────────────────────
30
+ SAMPLE_N_PER_COUNTRY = 100_000 # training sample size per country
31
+ TOP_K = 35 # max candidates per S1 entity
32
+ FREQ_CAP_DEFAULT = 500
33
+ FREQ_CAP_FRANCE = 1000
34
+ S1_BATCH_SIZE = 50_000 # S1 entities per DuckDB blocking batch
35
+
36
+ # Paths — auto-detect Kaggle vs local
37
+ DATA_DIR = None
38
+ for p in [Path("/kaggle/input/datasets/arghasaha1905/amazon-ml"),
39
+ Path("/kaggle/input/amazon-ml-challenge-2026"),
40
+ Path("/kaggle/input/amazon-ml"),
41
+ Path("dataset")]:
42
+ if p.exists():
43
+ DATA_DIR = p
44
+ break
45
+ assert DATA_DIR is not None, "Cannot find dataset directory"
46
+
47
+ OUT_DIR = Path("/kaggle/working/output") if Path("/kaggle/working").exists() else Path("output")
48
+ CACHE_DIR = Path("/kaggle/working/cache") if Path("/kaggle/working").exists() else Path("cache")
49
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
50
+ CACHE_DIR.mkdir(parents=True, exist_ok=True)
51
+
52
+ print(f"DATA_DIR = {DATA_DIR}")
53
+ print(f"OUT_DIR = {OUT_DIR}")
54
+ print(f"CACHE_DIR = {CACHE_DIR}")
55
+
56
+ # ─────────────────────────────────────────────────────────
57
+ # 1. TRANSLITERATION
58
+ # ─────────────────────────────────────────────────────────
59
+ DEV_PHONETICS = {
60
+ 0x05:'a',0x06:'a',0x07:'i',0x08:'i',0x09:'u',0x0A:'u',0x0B:'ri',
61
+ 0x0E:'e',0x0F:'e',0x10:'ai',0x12:'o',0x13:'o',0x14:'au',
62
+ 0x15:'k',0x16:'kh',0x17:'g',0x18:'gh',0x19:'n',
63
+ 0x1A:'ch',0x1B:'chh',0x1C:'j',0x1D:'jh',0x1E:'n',
64
+ 0x1F:'t',0x20:'th',0x21:'d',0x22:'dh',0x23:'n',
65
+ 0x24:'t',0x25:'th',0x26:'d',0x27:'dh',0x28:'n',
66
+ 0x2A:'p',0x2B:'ph',0x2C:'b',0x2D:'bh',0x2E:'m',
67
+ 0x2F:'y',0x30:'r',0x31:'r',0x32:'l',0x33:'l',0x35:'v',
68
+ 0x36:'sh',0x37:'sh',0x38:'s',0x39:'h',
69
+ 0x3E:'a',0x3F:'i',0x40:'i',0x41:'u',0x42:'u',0x43:'ri',
70
+ 0x46:'e',0x47:'e',0x48:'ai',0x4A:'o',0x4B:'o',0x4C:'au',
71
+ 0x4D:'',0x02:'n',0x01:'n',0x03:'h'
72
+ }
73
+ TRANSLIT_MAP = {chr(base+off): lat
74
+ for base in (0x0900,0x0980,0x0A00,0x0A80,0x0B00,
75
+ 0x0B80,0x0C00,0x0C80,0x0D00)
76
+ for off, lat in DEV_PHONETICS.items()}
77
+ TRANSLIT_MAP.update({
78
+ 'é':'e','è':'e','ê':'e','ë':'e','à':'a','â':'a','ä':'a',
79
+ 'ô':'o','ö':'o','î':'i','ï':'i','ù':'u','û':'u','ü':'u',
80
+ 'ç':'c','ñ':'n',
81
+ 'É':'e','È':'e','Ê':'e','Ë':'e','À':'a','Â':'a','Ä':'a',
82
+ 'Ô':'o','Ö':'o','Î':'i','Ï':'i','Ù':'u','Û':'u','Ü':'u',
83
+ 'Ç':'c','Ñ':'n',
84
+ '«':'','»':'','\u2018':"'",'\u201c':'"','\u201d':'"'
85
+ })
86
+ TRANSLIT_TABLE = str.maketrans(TRANSLIT_MAP)
87
+
88
+ SUFFIX_RE = re.compile(
89
+ r'\b(pvt|ltd|limited|private|inc|incorporated|llc|corp|corporation|'
90
+ r'co|company|enterprises|services|solutions|group|sarl|sas|sasu|'
91
+ r'eurl|sci|association|fils)\b', re.I)
92
+
93
+ _NONALNUM = re.compile(r'[^a-z0-9]+')
94
+
95
+ def fast_norm(value):
96
+ if not value or (isinstance(value, float) and value != value):
97
+ return ''
98
+ s = str(value).translate(TRANSLIT_TABLE)
99
+ s = unicodedata.normalize('NFKD', s)
100
+ s = ''.join(ch for ch in s if not unicodedata.combining(ch))
101
+ s = s.encode('ascii', 'ignore').decode('ascii').lower()
102
+ s = re.sub(r'\b(https?://|www\.)', ' ', s)
103
+ return _NONALNUM.sub(' ', s).strip()
104
+
105
+ def root_name(n):
106
+ return SUFFIX_RE.sub(' ', n).strip()
107
+
108
+ def extract_postal(addr):
109
+ m = re.search(r'\b\d{5,6}\b', addr or '')
110
+ return m.group(0) if m else ''
111
+
112
+ def extract_house(addr):
113
+ m = re.search(r'\b\d+[a-z]?\b', addr or '')
114
+ return m.group(0) if m else ''
115
+
116
+ # Sanity check
117
+ for raw in ["रियल कंसल्टेंट्स प्राइवेट लिमिटेड", "Thérapy", "பாலாஜி"]:
118
+ out = fast_norm(raw)
119
+ assert out != '', f"Transliteration FAILED for {raw!r}"
120
+ print("Transliteration sanity check: PASSED")
121
+
122
+
123
+ # ─────────────────────────────────────────────────────────
124
+ # 2. DATA LOADING
125
+ # ─────────────────────────────────────────────────────────
126
+ def find_file(name):
127
+ for sub in ['', 'train', 'test']:
128
+ p = DATA_DIR / sub / name if sub else DATA_DIR / name
129
+ if p.exists():
130
+ return p
131
+ raise FileNotFoundError(f"Cannot find {name} under {DATA_DIR}")
132
+
133
+ def load_tsv(name):
134
+ path = find_file(name)
135
+ df = pd.read_csv(path, sep='\t', dtype=str, keep_default_na=False)
136
+ print(f" {name}: {len(df):,} rows")
137
+ return df
138
+
139
+ print("\n=== Loading datasets ===")
140
+ t0_global = time.time()
141
+ train_s1 = load_tsv("train_source1.tsv")
142
+ train_s2 = load_tsv("train_source2.tsv")
143
+ train_s3 = load_tsv("train_source3.tsv")
144
+ train_gt = load_tsv("train_ground_truth.tsv")
145
+
146
+ # ─────────────────────────────────────────────────────────
147
+ # 3. SAMPLE + NORMALIZE
148
+ # ─────────────────────────────────────────────────────────
149
+ print(f"\n=== Sampling {SAMPLE_N_PER_COUNTRY:,} per country for training ===")
150
+ train_s1_sample = (train_s1
151
+ .groupby('country', group_keys=False)
152
+ .apply(lambda g: g.sample(min(SAMPLE_N_PER_COUNTRY, len(g)), random_state=42))
153
+ .reset_index(drop=True))
154
+ keep_ids = set(train_s1_sample['entity_id'])
155
+ train_gt_sample = train_gt[train_gt['source1_entity_id'].isin(keep_ids)].reset_index(drop=True)
156
+ print(f"Sampled train_s1: {len(train_s1_sample):,} rows")
157
+ print(f"Sampled train_gt: {len(train_gt_sample):,} rows")
158
+
159
+ def normalize_df(df, label=''):
160
+ t = time.time()
161
+ df = df.copy()
162
+ df['name_clean'] = [fast_norm(v) for v in df['business_name']]
163
+ df['name_root'] = [root_name(v) for v in df['name_clean']]
164
+ df['addr_clean'] = [fast_norm(v) for v in df['business_address']]
165
+ df['name_compact'] = [_NONALNUM.sub('', v) for v in df['name_clean']]
166
+ df['house_no'] = [extract_house(v) for v in df['addr_clean']]
167
+ df['postal'] = [extract_postal(v) for v in df['addr_clean']]
168
+ print(f" Normalized {label}: {len(df):,} rows in {time.time()-t:.1f}s")
169
+ return df
170
+
171
+ print("\nNormalizing train S1 sample...")
172
+ train_s1_sample = normalize_df(train_s1_sample, 'train_s1_sample')
173
+
174
+ empty_frac = (train_s1_sample['name_clean'] == '').mean()
175
+ print(f"Empty name_clean fraction: {empty_frac:.4%}")
176
+ assert empty_frac < 0.02, f"Too many empty names: {empty_frac:.2%}"
177
+
178
+ # ─────────────────────────────────────────────────────────
179
+ # 4. DuckDB BLOCKING
180
+ # ─────────────────────────────────────────────────────────
181
+ KEY_SQL_TEMPLATE = """
182
+ SELECT entity_id, 'p5:' || substr(name_compact, 1, 5) AS bkey FROM {t} WHERE length(name_compact) >= 5
183
+ UNION ALL
184
+ SELECT entity_id, 'nt:' || tok AS bkey
185
+ FROM (SELECT entity_id, unnest(string_split(name_clean, ' ')) AS tok FROM {t})
186
+ WHERE length(tok) >= 3
187
+ UNION ALL
188
+ SELECT entity_id, 'nb:' || toks[1] || '_' || toks[2] AS bkey
189
+ FROM (SELECT entity_id, string_split(name_clean, ' ') AS toks FROM {t})
190
+ WHERE len(toks) >= 2 AND length(toks[1]) >= 3 AND length(toks[2]) >= 3
191
+ UNION ALL
192
+ SELECT entity_id, 'np:' || substr(toks[1],1,3) || '_' || substr(toks[2],1,3) AS bkey
193
+ FROM (SELECT entity_id, string_split(name_clean, ' ') AS toks FROM {t})
194
+ WHERE len(toks) >= 2 AND length(toks[1]) >= 3 AND length(toks[2]) >= 3
195
+ UNION ALL
196
+ SELECT entity_id, 'at:' || tok AS bkey
197
+ FROM (SELECT entity_id, unnest(string_split(addr_clean, ' ')) AS tok FROM {t})
198
+ WHERE length(tok) >= 4
199
+ UNION ALL
200
+ SELECT entity_id, 'pn:' || postal || '_' || substr(name_compact,1,3) AS bkey
201
+ FROM {t} WHERE postal != '' AND length(name_compact) >= 3
202
+ UNION ALL
203
+ SELECT entity_id, 'hn:' || house_no || '_' || substr(name_compact,1,3) AS bkey
204
+ FROM {t} WHERE house_no != '' AND length(name_compact) >= 3
205
+ UNION ALL
206
+ SELECT entity_id, 'ht:' || house_no || '_' || toks[1] AS bkey
207
+ FROM (SELECT entity_id, house_no, string_split(name_clean, ' ') AS toks FROM {t})
208
+ WHERE house_no != '' AND len(toks) >= 1 AND length(toks[1]) >= 3
209
+ """
210
+
211
+ def block_country(con, s1_df, targets_df, country, freq_cap, top_k):
212
+ """Run DuckDB blocking for one country. Target keys indexed ONCE."""
213
+ t_start = time.time()
214
+ n_s1 = len(s1_df)
215
+ print(f"\n [{country}] S1={n_s1:,}, Targets={len(targets_df):,}")
216
+
217
+ con.register('tgt_raw', targets_df)
218
+ con.execute(f"CREATE OR REPLACE TEMP TABLE target_keys AS {KEY_SQL_TEMPLATE.format(t='tgt_raw')}")
219
+ con.execute(f"""
220
+ CREATE OR REPLACE TEMP TABLE filtered_tgt_keys AS
221
+ SELECT t.entity_id, t.bkey
222
+ FROM target_keys t
223
+ JOIN (SELECT bkey, count(*) AS n FROM target_keys GROUP BY bkey HAVING n <= {freq_cap}) f
224
+ ON t.bkey = f.bkey
225
+ """)
226
+ con.execute("CREATE INDEX IF NOT EXISTS ftk_idx ON filtered_tgt_keys(bkey);")
227
+ con.unregister('tgt_raw')
228
+
229
+ entity_ids = s1_df['entity_id'].tolist()
230
+ n_batches = (n_s1 + S1_BATCH_SIZE - 1) // S1_BATCH_SIZE
231
+ all_cands = []
232
+
233
+ for b in range(n_batches):
234
+ bt = time.time()
235
+ batch = s1_df.iloc[b * S1_BATCH_SIZE : (b + 1) * S1_BATCH_SIZE]
236
+ con.register('s1_b', batch)
237
+ con.execute(f"CREATE OR REPLACE TEMP TABLE s1_keys AS {KEY_SQL_TEMPLATE.format(t='s1_b')}")
238
+
239
+ joined = con.execute(f"""
240
+ WITH pair_hits AS (
241
+ SELECT s.entity_id AS s1_id, t.entity_id AS target_id, count(*) AS key_hits
242
+ FROM s1_keys s
243
+ JOIN filtered_tgt_keys t ON s.bkey = t.bkey
244
+ GROUP BY 1, 2
245
+ ),
246
+ ranked AS (
247
+ SELECT *, row_number() OVER(PARTITION BY s1_id ORDER BY key_hits DESC, target_id) AS rk
248
+ FROM pair_hits
249
+ )
250
+ SELECT s1_id, target_id, key_hits FROM ranked WHERE rk <= {top_k};
251
+ """).fetchdf()
252
+
253
+ con.unregister('s1_b')
254
+ all_cands.append(joined)
255
+ print(f" Batch {b+1}/{n_batches}: {len(batch):,} S1 -> {len(joined):,} pairs ({time.time()-bt:.1f}s)")
256
+
257
+ con.execute("DROP TABLE IF EXISTS target_keys; DROP TABLE IF EXISTS filtered_tgt_keys; DROP TABLE IF EXISTS s1_keys;")
258
+ result = pd.concat(all_cands, ignore_index=True) if all_cands else pd.DataFrame(columns=['s1_id','target_id','key_hits'])
259
+ print(f" [{country}] done: {len(result):,} pairs in {time.time()-t_start:.1f}s")
260
+ return result
261
+
262
+
263
+ def run_full_blocking(s1_df, s2_df, s3_df, label="train"):
264
+ """Run blocking across all countries, combining S2+S3 targets."""
265
+ print(f"\n{'='*60}")
266
+ print(f"BLOCKING ({label.upper()})")
267
+ print(f"{'='*60}")
268
+ t_block = time.time()
269
+ countries = sorted(s1_df['country'].unique())
270
+ all_frames = []
271
+
272
+ for ctry in countries:
273
+ con = duckdb.connect()
274
+ con.execute("PRAGMA threads=8;")
275
+ con.execute("PRAGMA max_memory='12GB';")
276
+
277
+ cap = FREQ_CAP_FRANCE if ctry == 'France' else FREQ_CAP_DEFAULT
278
+ s1_c = s1_df[s1_df['country'] == ctry]
279
+
280
+ # Normalize S2+S3 for this country and combine
281
+ s2_c = normalize_df(s2_df[s2_df['country'] == ctry], f'{label}_s2_{ctry}')
282
+ s3_c = normalize_df(s3_df[s3_df['country'] == ctry], f'{label}_s3_{ctry}')
283
+ targets_c = pd.concat([s2_c, s3_c], ignore_index=True)
284
+
285
+ cands = block_country(con, s1_c, targets_c, ctry, cap, TOP_K)
286
+ all_frames.append(cands)
287
+ con.close()
288
+ del s2_c, s3_c, targets_c, con
289
+ gc.collect()
290
+
291
+ result = pd.concat(all_frames, ignore_index=True)
292
+ print(f"\nTotal {label} candidate pairs: {len(result):,} in {time.time()-t_block:.1f}s")
293
+ return result
294
+
295
+
296
+ # ─────────────────────────────────────────────────────────
297
+ # 5. BLOCKING RECALL CEILING CHECK
298
+ # ─────────────────────────────────────────────────────────
299
+ def check_recall(candidates, gt_df):
300
+ gt_pairs = set()
301
+ for _, row in gt_df.iterrows():
302
+ s1 = row['source1_entity_id']
303
+ matched = str(row.get('matched_entity_ids', '') or '').strip()
304
+ if matched and matched != 'nan':
305
+ for t in matched.split(','):
306
+ t = t.strip()
307
+ if t:
308
+ gt_pairs.add((s1, t))
309
+ cand_set = set(zip(candidates['s1_id'], candidates['target_id']))
310
+ recalled = gt_pairs & cand_set
311
+ recall = len(recalled) / len(gt_pairs) if gt_pairs else 0.0
312
+ print(f"\n Ground truth pairs: {len(gt_pairs):,}")
313
+ print(f" Recalled by blocking: {len(recalled):,}")
314
+ print(f" BLOCKING RECALL CEILING: {recall:.4%}")
315
+ return recall
316
+
317
+
318
+ # ─────────────────────────────────────────────────────────
319
+ # 6. FEATURE EXTRACTION
320
+ # ─────────────────────────────────────────────────────────
321
+ FEATURE_NAMES = [
322
+ 'key_hits',
323
+ 'name_ratio', 'name_tsort', 'name_tset', 'name_partial',
324
+ 'root_ratio', 'root_tsort',
325
+ 'addr_ratio', 'addr_tsort', 'addr_tset', 'addr_partial',
326
+ 'postal_match', 'postal_conflict', 'house_match',
327
+ 'name_len_ratio', 'addr_token_overlap',
328
+ ]
329
+
330
+ def extract_features(candidates, s1_df, s2_df, s3_df):
331
+ """Extract 16 lexical/structural features for candidate pairs."""
332
+ t_feat = time.time()
333
+ print(f"\n Extracting features for {len(candidates):,} pairs...")
334
+
335
+ # Build lookup dicts
336
+ s1_dict = s1_df.set_index('entity_id')[['name_clean','name_root','addr_clean','house_no','postal','country']].to_dict('index')
337
+ tgt_df = pd.concat([s2_df, s3_df], ignore_index=True).drop_duplicates(subset='entity_id')
338
+ tgt_dict = tgt_df.set_index('entity_id')[['name_clean','name_root','addr_clean','house_no','postal','country']].to_dict('index')
339
+
340
+ n = len(candidates)
341
+ feat = np.zeros((n, len(FEATURE_NAMES)), dtype=np.float32)
342
+
343
+ s1_ids = candidates['s1_id'].values
344
+ tgt_ids = candidates['target_id'].values
345
+ hits = candidates['key_hits'].values.astype(np.float32)
346
+
347
+ for i in range(n):
348
+ s = s1_dict.get(s1_ids[i])
349
+ t = tgt_dict.get(tgt_ids[i])
350
+ if s is None or t is None:
351
+ feat[i, 0] = hits[i]
352
+ continue
353
+
354
+ sn, tn = s['name_clean'], t['name_clean']
355
+ sr, tr = s['name_root'], t['name_root']
356
+ sa, ta = s['addr_clean'], t['addr_clean']
357
+ sp, tp = s['postal'], t['postal']
358
+ sh, th = s['house_no'], t['house_no']
359
+
360
+ feat[i, 0] = hits[i]
361
+ feat[i, 1] = fuzz.ratio(sn, tn)
362
+ feat[i, 2] = fuzz.token_sort_ratio(sn, tn)
363
+ feat[i, 3] = fuzz.token_set_ratio(sn, tn)
364
+ feat[i, 4] = fuzz.partial_ratio(sn, tn)
365
+ feat[i, 5] = fuzz.ratio(sr, tr)
366
+ feat[i, 6] = fuzz.token_sort_ratio(sr, tr)
367
+ feat[i, 7] = fuzz.ratio(sa, ta)
368
+ feat[i, 8] = fuzz.token_sort_ratio(sa, ta)
369
+ feat[i, 9] = fuzz.token_set_ratio(sa, ta)
370
+ feat[i, 10] = fuzz.partial_ratio(sa, ta)
371
+ feat[i, 11] = 1.0 if (sp and tp and sp == tp) else 0.0
372
+ feat[i, 12] = 1.0 if (sp and tp and sp != tp) else 0.0
373
+ feat[i, 13] = 1.0 if (sh and th and sh == th) else 0.0
374
+
375
+ ln_s, ln_t = max(len(sn), 1), max(len(tn), 1)
376
+ feat[i, 14] = min(ln_s, ln_t) / max(ln_s, ln_t)
377
+
378
+ s_toks = set(sa.split()) if sa else set()
379
+ t_toks = set(ta.split()) if ta else set()
380
+ denom = max(len(s_toks | t_toks), 1)
381
+ feat[i, 15] = len(s_toks & t_toks) / denom
382
+
383
+ if (i + 1) % 500_000 == 0:
384
+ print(f" {i+1:,}/{n:,} features extracted ({time.time()-t_feat:.0f}s)")
385
+
386
+ print(f" Features done: {n:,} pairs in {time.time()-t_feat:.1f}s")
387
+ return feat
388
+
389
+
390
+ # ─────────────────────────────────────────────────────────
391
+ # 7. BUILD GROUND-TRUTH LOOKUP
392
+ # ─────────────────────────────────────────────────────────
393
+ gt_lookup = defaultdict(set)
394
+ for _, row in train_gt_sample.iterrows():
395
+ s1 = row['source1_entity_id']
396
+ matched = str(row.get('matched_entity_ids', '') or '').strip()
397
+ if matched and matched != 'nan':
398
+ for t in matched.split(','):
399
+ t = t.strip()
400
+ if t:
401
+ gt_lookup[s1].add(t)
402
+
403
+
404
+ # ─────────────────────────────────────────────────────────
405
+ # 8. METRIC
406
+ # ─────────────────────────────────────────────────────────
407
+ def macro_f05(eval_s1_set, s1_arr, tgt_arr, probs, threshold, gt_map):
408
+ """Compute Macro F0.5 strictly on eval_s1_set entities."""
409
+ preds = defaultdict(set)
410
+ for s, t, p in zip(s1_arr, tgt_arr, probs):
411
+ if p >= threshold:
412
+ preds[s].add(t)
413
+
414
+ scores = []
415
+ for s in eval_s1_set:
416
+ pred = preds.get(s, set())
417
+ true = gt_map.get(s, set())
418
+ if not pred and not true:
419
+ scores.append(1.0)
420
+ continue
421
+ tp = len(pred & true)
422
+ prec = tp / len(pred) if pred else 0.0
423
+ rec = tp / len(true) if true else 0.0
424
+ denom = 0.25 * prec + rec
425
+ f05 = (1.25 * prec * rec) / denom if denom > 0 else 0.0
426
+ scores.append(f05)
427
+ return float(np.mean(scores))
428
+
429
+
430
+ # ─────────────────────────────────────────────────────────
431
+ # MAIN PIPELINE
432
+ # ─────────────────────────────────────────────────────────
433
+ def main():
434
+ global train_s1_sample # need to access for country mapping later
435
+
436
+ # ── PHASE 1: Training Blocking ──
437
+ train_cands = run_full_blocking(train_s1_sample, train_s2, train_s3, label="train")
438
+
439
+ recall_ceil = check_recall(train_cands, train_gt_sample)
440
+ assert recall_ceil >= 0.80, f"Blocking recall {recall_ceil:.2%} too low!"
441
+
442
+ # ── PHASE 2: Training Features ──
443
+ print(f"\n{'='*60}")
444
+ print("FEATURE EXTRACTION (TRAIN)")
445
+ print(f"{'='*60}")
446
+
447
+ # Normalize targets that appear in candidate pairs
448
+ needed_tids = set(train_cands['target_id'])
449
+ s2_needed = normalize_df(train_s2[train_s2['entity_id'].isin(needed_tids)], 'train_s2_needed')
450
+ s3_needed = normalize_df(train_s3[train_s3['entity_id'].isin(needed_tids)], 'train_s3_needed')
451
+
452
+ X_train = extract_features(train_cands, train_s1_sample, s2_needed, s3_needed)
453
+ s1_arr = train_cands['s1_id'].values
454
+ tgt_arr = train_cands['target_id'].values
455
+
456
+ y_train = np.array([1 if t in gt_lookup[s] else 0 for s, t in zip(s1_arr, tgt_arr)], dtype=np.float32)
457
+ n_pos = int(y_train.sum())
458
+ n_neg = len(y_train) - n_pos
459
+ scale_pos_weight = n_neg / max(n_pos, 1)
460
+ print(f"\n Positives: {n_pos:,} | Negatives: {n_neg:,} | scale_pos_weight: {scale_pos_weight:.2f}")
461
+
462
+ del s2_needed, s3_needed
463
+ gc.collect()
464
+
465
+ # ── PHASE 3: LightGBM 5-Fold CV ──
466
+ print(f"\n{'='*60}")
467
+ print("LIGHTGBM 5-FOLD CV")
468
+ print(f"{'='*60}")
469
+ import lightgbm as lgb
470
+ from sklearn.model_selection import GroupKFold
471
+
472
+ gkf = GroupKFold(n_splits=5)
473
+ oof_pred = np.zeros(len(X_train))
474
+ fold_scores = []
475
+
476
+ params = dict(
477
+ objective='binary',
478
+ metric='binary_logloss',
479
+ scale_pos_weight=scale_pos_weight,
480
+ num_leaves=63,
481
+ learning_rate=0.05,
482
+ n_estimators=600,
483
+ min_child_samples=50,
484
+ subsample=0.8,
485
+ colsample_bytree=0.8,
486
+ n_jobs=-1,
487
+ verbosity=-1
488
+ )
489
+
490
+ for fold, (tr_idx, va_idx) in enumerate(gkf.split(X_train, y_train, s1_arr)):
491
+ tr_set = lgb.Dataset(X_train[tr_idx], label=y_train[tr_idx])
492
+ va_set = lgb.Dataset(X_train[va_idx], label=y_train[va_idx])
493
+
494
+ booster = lgb.train(params, tr_set, valid_sets=[va_set],
495
+ callbacks=[lgb.early_stopping(30, verbose=False)])
496
+ va_prob = booster.predict(X_train[va_idx])
497
+ oof_pred[va_idx] = va_prob
498
+
499
+ va_s1 = set(s1_arr[va_idx])
500
+ f05 = macro_f05(va_s1, s1_arr[va_idx], tgt_arr[va_idx], va_prob, 0.50, gt_lookup)
501
+ fold_scores.append(f05)
502
+ print(f" Fold {fold+1}: macro_F0.5 = {f05:.4f}")
503
+
504
+ mean_cv = np.mean(fold_scores)
505
+ print(f"\n CV Mean Macro F0.5: {mean_cv:.4f} +/- {np.std(fold_scores):.4f}")
506
+
507
+ # ── PHASE 4: Per-country threshold sweep ──
508
+ print(f"\n{'='*60}")
509
+ print("PER-COUNTRY THRESHOLD OPTIMIZATION")
510
+ print(f"{'='*60}")
511
+
512
+ s1_country = train_s1_sample.set_index('entity_id')['country'].to_dict()
513
+ cand_countries = np.array([s1_country.get(s, 'Unknown') for s in s1_arr])
514
+
515
+ per_country_threshold = {}
516
+ for ctry in np.unique(cand_countries):
517
+ mask = cand_countries == ctry
518
+ ctry_s1 = set(s1_arr[mask])
519
+ best_t, best_sc = 0.50, -1.0
520
+ for t in np.arange(0.30, 0.96, 0.02):
521
+ sc = macro_f05(ctry_s1, s1_arr[mask], tgt_arr[mask], oof_pred[mask], t, gt_lookup)
522
+ if sc > best_sc:
523
+ best_sc, best_t = sc, t
524
+ per_country_threshold[ctry] = round(float(best_t), 2)
525
+ print(f" {ctry}: threshold={best_t:.2f} F0.5={best_sc:.4f}")
526
+
527
+ default_t = 0.50
528
+ print(f" Default threshold: {default_t}")
529
+
530
+ # ── PHASE 5: Train final model ──
531
+ print(f"\n{'='*60}")
532
+ print("TRAINING FINAL MODEL ON ALL DATA")
533
+ print(f"{'='*60}")
534
+ full_set = lgb.Dataset(X_train, label=y_train)
535
+ final_model = lgb.train(params, full_set)
536
+ final_model.save_model(str(CACHE_DIR / 'model.lgb'))
537
+ print(" Final model saved.")
538
+
539
+ with open(CACHE_DIR / 'meta.json', 'w') as f:
540
+ json.dump({
541
+ 'feature_names': FEATURE_NAMES,
542
+ 'per_country_threshold': per_country_threshold,
543
+ 'default_threshold': default_t,
544
+ 'cv_mean_f05': float(mean_cv),
545
+ 'blocking_recall': float(recall_ceil),
546
+ }, f, indent=2)
547
+
548
+ del X_train, y_train, oof_pred, train_cands
549
+ gc.collect()
550
+
551
+ # ── PHASE 6: TEST INFERENCE ──
552
+ print(f"\n{'='*60}")
553
+ print("TEST INFERENCE")
554
+ print(f"{'='*60}")
555
+
556
+ print("\nLoading test data...")
557
+ test_s1 = load_tsv("test_source1.tsv")
558
+ test_s2 = load_tsv("test_source2.tsv")
559
+ test_s3 = load_tsv("test_source3.tsv")
560
+
561
+ print("\nNormalizing test S1...")
562
+ test_s1 = normalize_df(test_s1, 'test_s1')
563
+
564
+ all_test_s1_ids = test_s1['entity_id'].tolist()
565
+
566
+ # Run blocking on test set
567
+ test_cands = run_full_blocking(test_s1, test_s2, test_s3, label="test")
568
+
569
+ # Extract features per country to manage memory
570
+ print(f"\n{'='*60}")
571
+ print("FEATURE EXTRACTION (TEST)")
572
+ print(f"{'='*60}")
573
+
574
+ needed_test_tids = set(test_cands['target_id'])
575
+ test_s2_norm = normalize_df(test_s2[test_s2['entity_id'].isin(needed_test_tids)], 'test_s2_needed')
576
+ test_s3_norm = normalize_df(test_s3[test_s3['entity_id'].isin(needed_test_tids)], 'test_s3_needed')
577
+
578
+ X_test = extract_features(test_cands, test_s1, test_s2_norm, test_s3_norm)
579
+
580
+ del test_s2_norm, test_s3_norm
581
+ gc.collect()
582
+
583
+ # Predict
584
+ print("\n Running LightGBM prediction...")
585
+ test_probs = final_model.predict(X_test)
586
+
587
+ # Apply per-country thresholds
588
+ test_s1_ctry = test_s1.set_index('entity_id')['country'].to_dict()
589
+ test_s1_ids = test_cands['s1_id'].values
590
+ test_tgt_ids = test_cands['target_id'].values
591
+
592
+ thresholds = np.array([per_country_threshold.get(test_s1_ctry.get(s, ''), default_t) for s in test_s1_ids])
593
+ above = test_probs >= thresholds
594
+
595
+ print(f" Pairs above threshold: {above.sum():,} / {len(above):,} ({above.mean():.1%})")
596
+
597
+ # ── PHASE 7: 1:Many Target Exclusivity ──
598
+ print(f"\n{'='*60}")
599
+ print("TARGET EXCLUSIVITY RESOLUTION")
600
+ print(f"{'='*60}")
601
+
602
+ # Each target -> at most 1 S1 (highest prob); each S1 -> many targets
603
+ target_best = {} # target_id -> (s1_id, prob)
604
+ for i in range(len(test_s1_ids)):
605
+ if not above[i]:
606
+ continue
607
+ tid = test_tgt_ids[i]
608
+ sid = test_s1_ids[i]
609
+ prob = float(test_probs[i])
610
+ if tid not in target_best or prob > target_best[tid][1]:
611
+ target_best[tid] = (sid, prob)
612
+
613
+ # Build S1 -> list of targets
614
+ s1_matches = defaultdict(list)
615
+ for tid, (sid, _) in target_best.items():
616
+ s1_matches[sid].append(tid)
617
+
618
+ n_matched_s1 = len(s1_matches)
619
+ n_singletons = len(all_test_s1_ids) - n_matched_s1
620
+ total_links = sum(len(v) for v in s1_matches.values())
621
+ print(f" Matched S1 entities: {n_matched_s1:,}")
622
+ print(f" Singletons: {n_singletons:,}")
623
+ print(f" Total match links: {total_links:,}")
624
+
625
+ # ── PHASE 8: OUTPUT FILES ──
626
+ print(f"\n{'='*60}")
627
+ print("WRITING OUTPUT FILES")
628
+ print(f"{'='*60}")
629
+
630
+ # matching_results.tsv
631
+ match_path = OUT_DIR / 'matching_results.tsv'
632
+ with open(match_path, 'w', encoding='utf-8', newline='') as f:
633
+ f.write("source1_entity_id\tmatched_entity_ids\n")
634
+ for sid in all_test_s1_ids:
635
+ targets = s1_matches.get(sid, [])
636
+ f.write(f"{sid}\t{','.join(sorted(targets))}\n")
637
+ print(f" Wrote {match_path}")
638
+
639
+ # candidate_pairs.tsv
640
+ cand_path = OUT_DIR / 'candidate_pairs.tsv'
641
+ # Build per-S1 candidate lists from test_cands
642
+ s1_cand_map = defaultdict(list)
643
+ for sid, tid in zip(test_cands['s1_id'], test_cands['target_id']):
644
+ s1_cand_map[sid].append(tid)
645
+
646
+ with open(cand_path, 'w', encoding='utf-8', newline='') as f:
647
+ f.write("source1_entity_id\tcandidate_entity_ids\n")
648
+ for sid in all_test_s1_ids:
649
+ cids = s1_cand_map.get(sid, [])
650
+ f.write(f"{sid}\t{','.join(cids)}\n")
651
+ print(f" Wrote {cand_path}")
652
+
653
+ total_time = time.time() - t0_global
654
+ print(f"\n{'='*60}")
655
+ print(f"PIPELINE COMPLETE IN {total_time/60:.1f} MINUTES")
656
+ print(f"{'='*60}")
657
+
658
+
659
+ if __name__ == '__main__':
660
+ main()