Upload folder using huggingface_hub (part 5)

.gitattributes

@@ -48,3 +48,12 @@ model-3/work/results_A/model_bundle.joblib.bak_ef05 filter=lfs diff=lfs merge=lf
 model-4/.venv/bin/python filter=lfs diff=lfs merge=lfs -text
 model-4/.venv/bin/python3 filter=lfs diff=lfs merge=lfs -text
 model-4/.venv/bin/python3.12 filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/blobs/7c/7ccdaee489cbb709d4624beb0f0556fc5552e6759bde1d881fca5c7a9ee5462c filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/blobs/99/994dbaac4eaaaf1033ce39ecedd612e77b89dbfe8efd8de25ca24bea40caf6bc filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/blobs/c4/c4b914ec5ab295fbee3b0e0fe62f22febb3c09d34dd3778745e0de034a76df33 filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3 filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959 filter=lfs diff=lfs merge=lfs -text
+model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead filter=lfs diff=lfs merge=lfs -text
+model-4/output/candidate_pairs.tsv filter=lfs diff=lfs merge=lfs -text
+model-4/output/matching_results.tsv filter=lfs diff=lfs merge=lfs -text
+model-4/work/ce/final/tokenizer.json filter=lfs diff=lfs merge=lfs -text

model-4/Colab/src/text_utils.py

@@ -0,0 +1,264 @@
+"""Country-agnostic normalisation. No hand-written dictionaries: legal forms are discovered per country
+from unlabeled names, abbreviations are matched by a generic prefix/subsequence rule."""
+import math
+import re
+from collections import Counter
+from itertools import combinations
+
+from anyascii import anyascii
+from rapidfuzz.distance import JaroWinkler
+
+_PRE = str.maketrans({"°": "o ", "º": "o ","’": "'", "`": "'", " ": " "})
+_DOTTED = re.compile(r"\b(?:[a-z]\.){2,}")
+_NONALNUM = re.compile(r"[^a-z0-9]+")
+_DOMAIN = re.compile(r"^(?:www\.)?([a-z0-9][a-z0-9\-]*)\.[a-z]{2,4}$")
+_JOIN_ALNUM = re.compile(r"(?<=[a-z0-9])[-/](?=[a-z0-9])")
+_HOUSE_SEG = re.compile(r"[^a-z0-9#\-/. ]")
+_HOUSE = re.compile(r"^(?:#|[a-z]{1,3}\.?)?\s*(\d{1,6})(?:\s*-\s*(\d{1,6})\b|([a-z])\b|\s*/\s*([a-z0-9]{1,3})\b)?")
+_DIGITS = re.compile(r"\d+")
+_LETTER = re.compile(r"[a-z]")
+
+
+def is_nonlatin(s):
+    return any(ord(ch) > 0x24F for ch in s) if s else False
+
+
+def ascii_lower(s):
+    if not s:
+        return ""
+    return anyascii(s.translate(_PRE)).lower()
+
+
+def norm(s_ascii_lower):
+    s = _DOTTED.sub(lambda m: m.group(0).replace(".", ""), s_ascii_lower)
+    return _NONALNUM.sub(" ", s).strip()
+
+
+# ----------------------------------------------------------------------------------------------- names
+
+def name_tokens(raw):
+    a = ascii_lower(raw).strip()
+    if a and " " not in a:
+        m = _DOMAIN.match(a)
+        if m:
+            a = m.group(1)
+    return norm(a).split()
+
+
+def discover_legal_forms(token_lists, min_frac, boundary_ratio, max_cooc, bigram_tight):
+    """Legal forms = spans that sit at a name boundary, are frequent, and are mutually exclusive with each
+    other (SARL/SAS/EURL never co-occur, while descriptors like 'groupe' co-occur with a legal form)."""
+    n = len(token_lists)
+    bcount, total_uni = Counter(), Counter()
+    for t in token_lists:
+        total_uni.update(set(t))
+        if len(t) < 2:
+            continue
+        spans = {(t[-1],), (t[0],)}
+        if len(t) > 2:
+            spans |= {tuple(t[-2:]), tuple(t[:2])}
+        bcount.update(spans)
+    cands = [s for s, c in bcount.items() if c >= min_frac * n]
+    cset = set(cands)
+    total, occs, nbr = Counter(), [], {}
+    for t in token_lists:
+        if len(t) >= 2:
+            for k in (1, 2):
+                if len(t) > k:
+                    for s, nb in ((tuple(t[-k:]), t[-k - 1]), (tuple(t[:k]), t[k])):
+                        if s in cset:
+                            nbr.setdefault(s, Counter())[nb] += 1
+        occ = []
+        for k in (1, 2):
+            for i in range(len(t) - k + 1):
+                s = tuple(t[i:i + k])
+                if s in cset:
+                    occ.append((s, i, i + k))
+        if occ:
+            total.update({o[0] for o in occ})
+            occs.append(occ)
+    cands = [s for s in cands if bcount[s] >= boundary_ratio * total[s]]
+    # bigram must be "tight": its first token rarely appears outside it (private->private limited)
+    cands = [s for s in cands if len(s) == 1 or total[s] >= bigram_tight * total_uni[s[0]]]
+    # a legal form attaches to many different words; 'notre' is nearly always next to 'sainte'
+    in_bigram = {x for s in cands if len(s) == 2 for x in s}
+
+    def dominated(s):
+        c = nbr.get(s)
+        return bool(c) and s[0] not in in_bigram and c.most_common(1)[0][1] > 0.5 * sum(c.values())
+
+    cands = [s for s in cands if not dominated(s)]
+    keep = set(cands)
+    co = Counter()
+    for occ in occs:
+        occ = [o for o in occ if o[0] in keep]
+        for (a, ai, aj), (b, bi, bj) in combinations(occ, 2):
+            if a != b and (aj <= bi or bj <= ai):
+                co[frozenset((a, b))] += 1
+    accepted = []
+    for s in sorted(cands, key=lambda x: -bcount[x]):
+        if all(co[frozenset((s, a))] <= max_cooc * min(total[s], total[a]) for a in accepted):
+            accepted.append(s)
+    return sorted(accepted, key=lambda x: (-len(x), x))
+
+
+def strip_legal(tokens, forms, long_single):
+    t, found = list(tokens), []
+    changed = True
+    while changed and len(t) > 1:
+        changed = False
+        for f in forms:
+            k = len(f)
+            if len(t) > k and tuple(t[-k:]) == f:
+                found.append(" ".join(f)); del t[-k:]; changed = True; break
+            if len(t) > k and tuple(t[:k]) == f:
+                found.append(" ".join(f)); del t[:k]; changed = True; break
+    if len(t) > 1:
+        t2 = [x for x in t if x not in long_single]
+        if t2 and len(t2) < len(t):
+            found += [x for x in t if x in long_single]
+            t = t2
+    return t, " ".join(sorted(found))
+
+
+def process_name(raw, forms, long_single):
+    toks = name_tokens(raw)
+    core, legal = strip_legal(toks, forms, long_single)
+    return {"name_full": " ".join(toks), "name_core": " ".join(core), "legal": legal,
+            "name_ns": "".join(core), "nonlatin": is_nonlatin(raw)}
+
+
+# ----------------------------------------------------------------------------------------------- addresses
+
+def parse_address(raw):
+    raw = raw or ""
+    segs = [s for s in re.split(r"[,;]", raw) if s.strip()]
+    house, street_i, street_rest = "", -1, None
+    for i, seg in enumerate(segs):
+        s = _HOUSE_SEG.sub(" ", ascii_lower(seg)).strip()
+        m = _HOUSE.match(s)
+        if m and _LETTER.search(s[m.end():]):
+            base, hi, suf1, suf2 = m.group(1), m.group(2), m.group(3), m.group(4)
+            house = f"{int(base)}|{suf1 or suf2 or ''}|{int(hi) if hi else ''}"
+            street_i, street_rest = i, s[m.end():]
+            break
+    if street_i < 0 and segs:
+        street_i, street_rest = 0, ascii_lower(segs[0])
+    street = norm(street_rest or "")
+    loc = [norm(ascii_lower(s)) for j, s in enumerate(segs) if j != street_i]
+    loc = [x for x in loc if x]
+    al = ascii_lower(raw)
+    full = norm(al)
+    nums = _DIGITS.findall(full)
+    hb = house.split("|")[0] if house else None
+    postal = sorted({x for x in nums if len(x) in (5, 6) and x != hb})
+    alnum = sorted({t for t in norm(_JOIN_ALNUM.sub("", al)).split() if _LETTER.search(t) and any(c.isdigit() for c in t)})
+    return {"addr_norm": full, "street": street, "street_ns": street.replace(" ", ""), "loc": "|".join(loc),
+            "house": house, "nums": " ".join(sorted(set(nums))), "postal": " ".join(postal),
+            "alnum": " ".join(alnum), "addr_empty": not full}
+
+
+def house_compare(ha, hb):
+    """-> (state, log abs diff). 0 both missing, 1 one missing, 2 equal, 3 base equal/suffix differs,
+    4 range contains, 5 conflict."""
+    if not ha and not hb:
+        return 0, math.nan
+    if not ha or not hb:
+        return 1, math.nan
+    a, b = ha.split("|"), hb.split("|")
+    ab, bb = int(a[0]), int(b[0])
+    if ab == bb:
+        return (2 if a[1] == b[1] else 3), 0.0
+    if (a[2] and ab <= bb <= int(a[2])) or (b[2] and bb <= ab <= int(b[2])):
+        return 4, 0.0
+    return 5, math.log1p(abs(ab - bb))
+
+
+def set_state(a, b):
+    """0 both missing, 1 one missing, 2 share, 3 both present disjoint same lengths, 4 disjoint + length differs."""
+    if not a and not b:
+        return 0
+    if not a or not b:
+        return 1
+    if a & b:
+        return 2
+    return 4 if {len(x) for x in a}.isdisjoint({len(x) for x in b}) else 3
+
+
+# ----------------------------------------------------------------------------------------------- token matching
+
+def is_subseq(short, long):
+    it = iter(long)
+    return all(c in it for c in short)
+
+
+def abbrev_compat(a, b):
+    if a == b:
+        return True
+    s, l = (a, b) if len(a) <= len(b) else (b, a)
+    return len(s) <= 4 and len(l) > len(s) and s[0] == l[0] and not s.isdigit() and is_subseq(s, l)
+
+
+def tok_sim(a, b):
+    if a == b:
+        return 1.0
+    if a.isdigit() or b.isdigit():
+        return 0.0
+    if abbrev_compat(a, b):
+        return 0.9
+    if len(a) >= 4 and len(b) >= 4:
+        s = JaroWinkler.similarity(a, b)
+        if s >= 0.9:
+            return 0.9 * s
+    return 0.0
+
+
+def soft_cov(ta, tb):
+    """Fraction of ta covered by tb (exact / abbreviation / typo tolerant)."""
+    if not ta:
+        return math.nan
+    if not tb:
+        return 0.0
+    sb = set(tb)
+    tot = 0.0
+    for x in ta:
+        if x in sb:
+            tot += 1.0
+            continue
+        best = 0.0
+        for y in tb:
+            s = tok_sim(x, y)
+            if s > best:
+                best = s
+        tot += best
+    return tot / len(ta)
+
+
+def contains_seq(a, b):
+    k = len(a)
+    if k == 0 or k > len(b):
+        return False
+    return any(b[i:i + k] == a for i in range(len(b) - k + 1))
+
+
+def acronym_match(ta, tb):
+    if len(ta) >= 2:
+        ini = "".join(t[0] for t in ta)
+        if ini in tb or ini == "".join(tb):
+            return True
+    return False
+
+
+def legal_state(la, lb):
+    if not la and not lb:
+        return 0
+    if not la or not lb:
+        return 1
+    if la == lb:
+        return 2
+    ta, tb = la.split(), lb.split()
+    if len(ta) == len(tb) and all(abbrev_compat(x, y) for x, y in zip(ta, tb)):
+        return 3
+    if abbrev_compat(la.replace(" ", ""), lb.replace(" ", "")):
+        return 3
+    return 4

model-4/Colab/utils/validate_submission.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""
+ML Challenge 2026 — Submission Validator
+
+Run this BEFORE submitting. It checks your output files against every formatting
+rule the scorer enforces, so you can catch a rejection locally instead of burning
+a submission. It reads only your output files and the test source files (to learn
+which S1 entities are required and which S2/S3 IDs exist); it never needs the
+ground truth and never computes your score.
+
+It validates two files:
+
+* ``matching_results.tsv`` (required) — your final matches, the file scored on the
+  leaderboard.
+* ``candidate_pairs.tsv`` (optional) — the candidate set from your blocking stage.
+  When present, the validator also checks that your final matches are a subset of
+  your candidates and *warns* (never fails) otherwise. When absent it is skipped
+  with a warning; it is still expected in your final submission zip.
+
+Stdlib only, Python 3.8+. Run from the ``student_resource/`` directory::
+
+    python3 utils/validate_submission.py \
+        --matching output/matching_results.tsv \
+        --candidate output/candidate_pairs.tsv \
+        --test-dir dataset/test
+
+Exit code 0 means the files are safe to submit; 1 means fix the listed issues
+(warnings never fail the run).
+
+ID-existence check (off by default). By default the validator does NOT check that
+every matched/candidate ID actually exists in the test set: that check loads all
+Source-2/3 IDs into memory, which on the full ~1.7M-entity test set costs a few GB
+(more when ``candidate_pairs.tsv`` is included). The default run therefore stays fast
+and light and verifies every other rule; it prints a warning noting the check was
+skipped. Pass ``--check-ids`` to turn it on (it reads ``test_source2.tsv`` /
+``test_source3.tsv`` from ``--test-dir``); a missing/garbage matched ID only lowers
+your score rather than being rejected by the scorer, so this check is a diagnostic,
+not a gate. If ``--check-ids`` runs out of memory, drop ``--candidate`` (the candidate
+cross-check is the biggest memory user, and the matching file is the only one scored).
+"""
+
+import argparse
+import os
+import sys
+
+DELIM = "\t"
+MAX_EXAMPLES = 5  # how many offending IDs to show per issue
+MATCHING_HEADER = ["source1_entity_id", "matched_entity_ids"]
+CANDIDATE_HEADER = ["source1_entity_id", "candidate_entity_ids"]
+
+
+def read_ids(path):
+    """Return the set of first-column entity IDs from a source TSV.
+
+    The header row is skipped and blank lines are ignored.
+    """
+    with open(path, encoding="utf-8") as f:
+        next(f, None)  # skip header
+        return {line.split(DELIM, 1)[0].strip() for line in f if line.strip()}
+
+
+def examples(items):
+    """Return a short, human-readable sample of ``items`` for an error message."""
+    items = sorted(items)
+    shown = ", ".join(items[:MAX_EXAMPLES])
+    if len(items) > MAX_EXAMPLES:
+        return f"{len(items)} total, e.g. {shown}, ..."
+    return shown
+
+
+def load_match_targets(test_dir, warnings):
+    """Return the set of valid S2/S3 match IDs, or ``None`` if unavailable.
+
+    Only called when ``--check-ids`` is on. When ``test_source2.tsv`` or
+    ``test_source3.tsv`` is missing we cannot check that matched IDs exist, so we
+    record a warning and return ``None`` to signal that the existence check should be
+    skipped.
+    """
+    targets = set()
+    for name in ("test_source2.tsv", "test_source3.tsv"):
+        path = os.path.join(test_dir, name)
+        if not os.path.isfile(path):
+            warnings.append(
+                f"{path} not found — skipping the (optional) check that matched "
+                f"IDs exist in the test set. Every other rule is still checked. "
+                f"This is the lighter-memory mode; provide test_source2/3.tsv to "
+                f"enable the ID-existence check."
+            )
+            return None
+        targets |= read_ids(path)
+    return targets
+
+
+def validate_id_list_file(path, expected_header, col_label, required, valid_ids, errors):
+    """Validate one results-style TSV (matching or candidate).
+
+    Applies the shared formatting rules and appends any problems to ``errors``.
+    Returns a ``{source1_id: set(matched/candidate ids)}`` mapping, or ``None`` on a
+    fatal problem (missing file, empty file, or a broken header) that stops parsing.
+    """
+    if not os.path.isfile(path):
+        errors.append(f"File not found: {path}")
+        return None
+
+    name = os.path.basename(path)
+    mapping = {}
+    seen, dup_rows, intra_dupes = set(), set(), set()
+    self_matches, wrong_prefix, unknown = set(), set(), set()
+    n_rows = empties = 0
+
+    with open(path, encoding="utf-8") as f:
+        header = f.readline()
+        if not header:
+            errors.append(f"{name} is empty.")
+            return None
+        if DELIM not in header and "," in header:  # the #1 mistake: a CSV
+            errors.append(
+                f"{name}: header has no TAB but contains commas — the file looks "
+                "COMMA-separated. Submissions must be TAB-separated (.tsv); "
+                "write it with df.to_csv(sep='\\t', index=False)."
+            )
+            return None
+        cols = [c.strip().lower() for c in header.rstrip("\n").split(DELIM)]
+        if cols != expected_header:
+            errors.append(
+                f"{name}: unexpected header {cols}. "
+                f"Expected exactly {expected_header} (tab-separated)."
+            )
+            return None
+
+        for line_num, line in enumerate(f, start=2):
+            s1, tab, rest = line.partition(DELIM)
+            if not tab:
+                if s1.strip():
+                    errors.append(
+                        f"{name}: malformed row (no tab) at line {line_num}: "
+                        f"{line.rstrip()!r}"
+                    )
+                continue
+
+            n_rows += 1
+            if s1 in seen:
+                dup_rows.add(s1)
+            seen.add(s1)
+
+            ids = rest.rstrip("\n").split(",") if rest.strip() else []
+            if not ids:
+                empties += 1
+                mapping[s1] = set()
+                continue
+            if len(ids) != len(set(ids)):
+                intra_dupes.add(s1)
+            id_set = set(ids)
+            mapping[s1] = id_set
+            for mid in id_set:
+                if mid.startswith("S1-"):
+                    self_matches.add(mid)
+                elif not mid.startswith(("S2-", "S3-")):
+                    wrong_prefix.add(mid)
+                elif valid_ids is not None and mid not in valid_ids:
+                    unknown.add(mid)
+
+    # Aggregate the per-category findings. Each entry is (offenders, message);
+    # only non-empty categories become errors.
+    findings = [
+        (
+            dup_rows,
+            "{name}: duplicate source1_entity_id row(s): {ex}. "
+            "Each S1 entity may appear on only one row.",
+        ),
+        (
+            intra_dupes,
+            "{name}: repeated ID inside a {col} list for: {ex}. "
+            "No duplicate IDs are allowed within a list.",
+        ),
+        (
+            self_matches,
+            "{name}: {col} contains Source-1 IDs (self-matches): {ex}. "
+            "Only S2-/S3- IDs are allowed.",
+        ),
+        (
+            wrong_prefix,
+            "{name}: {col} contains IDs without an S2-/S3- prefix: {ex}.",
+        ),
+        (
+            unknown,
+            "{name}: {col} references IDs not in the test "
+            "Source-2/3 files: {ex}.",
+        ),
+        (
+            required - seen,
+            "{name}: required S1 entity(ies) missing: {ex}. "
+            "Every entity in test_source1.tsv needs a row (empty = no match).",
+        ),
+        (
+            seen - required,
+            "{name}: row(s) using an S1 ID that is not in the test set: {ex}.",
+        ),
+    ]
+    for offenders, message in findings:
+        if offenders:
+            errors.append(message.format(name=name, ex=examples(offenders), col=col_label))
+
+    print(f"  {name}: {n_rows} rows ({empties} empty, {n_rows - empties} non-empty).")
+    return mapping
+
+
+def validate(matching_path, candidate_path, test_dir, check_ids=False):
+    """Validate the submission output(s); return ``(errors, warnings)`` lists.
+
+    ``check_ids`` (``--check-ids``) turns on the optional, memory-heavy check that
+    every matched/candidate ID exists in the test Source-2/3 files. It is off by
+    default so the common run stays fast and light.
+    """
+    errors, warnings = [], []
+
+    source1 = os.path.join(test_dir, "test_source1.tsv")
+    if not os.path.isfile(source1):
+        errors.append(f"Test source1 file not found: {source1} (check --test-dir).")
+        return errors, warnings
+    required = read_ids(source1)
+    print(f"  required S1 entities: {len(required)}")
+
+    if check_ids:
+        valid_ids = load_match_targets(test_dir, warnings)
+        if valid_ids is not None:
+            print(f"  valid S2/S3 match IDs: {len(valid_ids)}")
+    else:
+        valid_ids = None
+        warnings.append(
+            "ID-existence check is OFF (the default) — not checking that matched/"
+            "candidate IDs exist in the test set. Every other rule is still checked. "
+            "Re-run with --check-ids to enable it (needs test_source2/3.tsv; uses "
+            "more memory). A nonexistent ID only lowers your score, never rejects "
+            "your submission."
+        )
+
+    matched = validate_id_list_file(
+        matching_path, MATCHING_HEADER, "matched_entity_ids", required, valid_ids, errors
+    )
+
+    # candidate_pairs.tsv is optional: if it's absent we skip its checks with a
+    # warning (it's still expected in your final submission zip). A missing
+    # candidate file never fails this run on its own.
+    candidate = None
+    if candidate_path and os.path.isfile(candidate_path):
+        candidate = validate_id_list_file(
+            candidate_path, CANDIDATE_HEADER, "candidate_entity_ids",
+            required, valid_ids, errors,
+        )
+    elif candidate_path:
+        warnings.append(
+            f"{candidate_path} not found — skipping candidate_pairs.tsv checks. "
+            "It is optional here, but your final submission zip must include "
+            "output/candidate_pairs.tsv."
+        )
+
+    # Soft check: your final matches should come from your blocking candidates.
+    # A matched ID absent from candidate_pairs.tsv usually means a pipeline bug,
+    # so we warn but never fail on it.
+    if matched is not None and candidate is not None:
+        offenders = {
+            s1 for s1, mids in matched.items() if mids - candidate.get(s1, set())
+        }
+        if offenders:
+            warnings.append(
+                f"{len(offenders)} S1 entity(ies) have matched IDs not present in "
+                f"candidate_pairs.tsv, e.g. {examples(offenders)}. Final matches "
+                "normally come from your blocking candidates — double-check these."
+            )
+
+    return errors, warnings
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Validate ML Challenge 2026 submission output files before submitting."
+    )
+    parser.add_argument(
+        "--matching",
+        "-m",
+        default="output/matching_results.tsv",
+        help="Path to matching_results.tsv (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--candidate",
+        "-c",
+        default=None,
+        help="Path to candidate_pairs.tsv "
+        "(default: output/candidate_pairs.tsv if it exists).",
+    )
+    parser.add_argument(
+        "--test-dir",
+        "-t",
+        default="dataset/test",
+        help="Folder with test_source1/2/3.tsv (default: %(default)s). "
+        "test_source2/3.tsv are only read when --check-ids is given.",
+    )
+    parser.add_argument(
+        "--check-ids",
+        action="store_true",
+        help="Also check that every matched/candidate ID exists in the test "
+        "Source-2/3 files. Off by default (loads all S2/S3 IDs into memory — a few "
+        "GB on the full test set). A nonexistent ID only lowers your score, so this "
+        "is a diagnostic, not a submission gate.",
+    )
+    args = parser.parse_args()
+
+    # candidate_pairs.tsv is optional; default to the conventional path and let
+    # validate() skip (with a warning) if the file isn't there.
+    candidate_path = args.candidate or "output/candidate_pairs.tsv"
+
+    print("ML Challenge 2026 — submission validator")
+    print(f"  test dir: {args.test_dir}")
+    try:
+        errors, warnings = validate(
+            args.matching, candidate_path, args.test_dir, check_ids=args.check_ids
+        )
+    except UnicodeDecodeError:
+        print()
+        print("FAIL — 1 issue(s) to fix before submitting:")
+        print(
+            f"  1. A file is not valid UTF-8 text (most likely {args.matching} or "
+            f"{candidate_path}). Re-save it as a plain UTF-8, tab-separated .tsv — "
+            "not cp1252/Latin-1, and not a compressed or binary file (.gz/.xlsx/"
+            ".parquet) renamed to .tsv. In pandas: "
+            "df.to_csv(path, sep='\\t', index=False, encoding='utf-8')."
+        )
+        return 1
+    except OSError as exc:
+        print()
+        print("FAIL — 1 issue(s) to fix before submitting:")
+        print(f"  1. Could not read a file: {exc}.")
+        return 1
+
+    print()
+    for warning in warnings:
+        print(f"WARNING: {warning}")
+    if errors:
+        print(f"FAIL — {len(errors)} issue(s) to fix before submitting:")
+        for i, error in enumerate(errors, 1):
+            print(f"  {i}. {error}")
+        return 1
+    print("PASS — no blocking issues found. Safe to submit.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

model-4/DESIGN.md

@@ -0,0 +1,220 @@
+# Business Entity Resolution — Multi-Embedding Pipeline
+
+For every Source 1 business, find the Source 2 / Source 3 records that describe the same
+real-world business, maximizing macro-averaged F0.5 (precision-weighted, singletons included).
+Training data covers US and India; the test set adds France, unseen at training time — nothing
+in this pipeline is hard-coded to a specific country.
+
+This document is self-contained: everything needed to understand and run this pipeline is here,
+no other project or prior approach required.
+
+---
+
+## 1. Problem recap
+
+- Three sources of business records (`S1-`, `S2-`, `S3-` entity IDs), fields: `business_name`,
+  `business_address`, `country`. Source 1 is the deduplicated reference; for every S1 entity we
+  must find all S2/S3 records describing the same business (zero, one, or many).
+- **Metric:** F0.5 per S1 entity, macro-averaged across all S1 entities. A singleton (no true
+  match) scores 1.0 for a correct empty prediction and 0.0 for any predicted match. F0.5 weights
+  precision twice as heavily as recall.
+- **Hard rules:** models must be MIT/Apache-2.0 licensed and ≤8B parameters; no external lookups,
+  APIs, or geocoding; country is an open string set, never filtered or one-hot encoded; ground
+  truth learning (abbreviation maps, calibration, thresholds) must never leak across folds.
+
+## 2. Architecture
+
+```
+raw TSVs (S1/S2/S3, train + test)
+        │
+        ▼
+[0] Integrity manifest — row counts, ID prefixes, duplicate IDs, embedding-row alignment
+    for every embedding source, ground-truth target coverage, S2/S3 multi-owner check
+        │
+        ▼
+[1] Normalisation — casefold, accent-fold, punctuation/number normalisation, legal-suffix and
+    abbreviation maps learned per fold from training positives only. Two parallel text views are
+    kept everywhere:
+      • rule-free  (core, addr, name_plain) → used for every FEATURE, so all countries
+        (including unseen France) look the same to the model
+      • rule-applied (bcore, baddr)         → used ONLY for blocking/retrieval, where the
+        learned abbreviation expansions help recall
+    France gets label-free suffix discovery at test time (no supervised France examples exist).
+        │
+        ▼
+[2] Splits — disjoint S1 entity groups: ce (learns rules + trains the cross-encoder), stack
+    (trains LightGBM, calibration, decision tuning), val (evaluation only). Split A: random
+    mixed-country groups. Split B: leave-one-country-out (train US → validate India, and the
+    reverse) as the proxy for how the model will behave on unseen France. Every split searches
+    the SAME shared S2+S3 pool — never partitioned — so a split's own true matches can never be
+    stranded on the wrong side, and validation faces the same one-shared-pool competition test
+    inference will face.
+        │
+        ▼
+[3] Candidate generation — multiple independent retrievers, fused by rank
+    ┌───────────────────────────────────────────────────────────────────┐
+    │  per-country partitioned, run independently over the S2∪S3 pool:  │
+    │                                                                   │
+    │   • Dense retriever: intfloat/multilingual-e5-large (MIT, 560M)   │
+    │   • Dense retriever: BAAI/bge-m3 (MIT, 568M)                      │
+    │   • Lexical retriever: inverted index over rare name tokens,      │
+    │     name prefixes, and address tokens (with learned abbreviation  │
+    │     maps applied to widen recall)                                 │
+    │                                                                   │
+    │  each is an EXACT cosine top-K search (GPU, bounded memory) or    │
+    │  exact inverted-index lookup — no approximate search              │
+    └───────────────────────────────────────────────────────────────────┘
+        │
+        ▼  reciprocal-rank fusion (RRF) decides which candidates SURVIVE THE CAP —
+           RRF(c) = Σ over retrievers that found c of  1 / (k0 + rank)
+           capped at ~100-150 candidates per S1 (tuned from recall-vs-K probes)
+        │
+        ▼  every surviving candidate keeps EACH retriever's own score, rank, gap-to-best,
+           and percentile as SEPARATE features — RRF only gates survival, the model never
+           sees a blended score
+        │
+        ▼
+[4] Pair features — ~35 country-agnostic features on the rule-free text:
+      Retrieval:      per-source score/rank/gap/percentile (e5, bge, lexical),
+                      retrieval_support_count (how many of the 3 independently found it)
+      Name:           Jaro-Winkler, Levenshtein, token-sort/set ratio, plain-name ratio, token
+                      Jaccard, dictionary-free abbreviation match, legal-suffix agreement
+      Address:        ratio/token-set/partial similarity, token Jaccard, exact house/unit
+                      alnum-token overlap, numeric-set overlap
+      Contradictions: graded, 3-state (missing/uncertain ≠ conflict — never conflated):
+                      general numeric conflict, postal-code-length conflict, house-number
+                      conflict
+      Cross-source:   triangle S2↔S3 corroboration for the same S1 (feature only — never
+                      lets the decision layer add a candidate blocking never retrieved)
+      Context:        same-country flag, name/address length, missingness flags, script flags
+        │
+        ▼
+[5] Cross-encoder — microsoft/mdeberta-v3-base (MIT), fine-tuned on ce-split pairs (positives +
+    hard negatives mined from blocking, both label orders, bf16). Its probability is stacked into
+    LightGBM as a feature, computed out-of-fold on the training side to avoid leakage.
+        │
+        ▼
+[6] Stacker — LightGBM, grouped K-fold on the stack split, early stopping on an inner holdout,
+    fold-averaged predictions. The variant (with/without cross-encoder) and decision family
+    (threshold vs. expected-F0.5) are chosen on the stack partition's own score only — the val
+    part is never used for any selection, only for reporting.
+        │
+        ▼
+[7] Calibration — cross-fitted isotonic regression, fit on the FULL out-of-fold stack scores,
+    decoupled from conflict resolution (calibrating after arbitration on a biased survivor set
+    silently miscalibrates scores — this pipeline avoids that ordering entirely).
+        │
+        ▼
+[8] Decision — exact expected-F0.5 selection per S1, then one-owner arbitration. For each S1,
+    for every candidate-set size k = 0, 1, 2, … (candidates sorted by calibrated probability),
+    the EXACT expected F0.5 of taking the top-k is computed via a Poisson-binomial dynamic
+    program (k=0 is how singletons earn their 1.0). Each entity independently picks its
+    best k; when two S1 entities claim the same S2/S3 record, the higher-scoring claim keeps it
+    and the loser re-decides without that candidate — iterated to a fixpoint, never a one-shot
+    greedy reservation.
+        │
+        ▼
+[9] Output + validation — matching_results.tsv and candidate_pairs.tsv, with in-script
+    duplicate/subset/ID checks, plus an optional call to the organizers' validate_submission.py
+    as the final gate before upload.
+```
+
+## 3. Why two dense retrievers, fused by rank rather than merged score
+
+Cosine similarity from two different embedding models isn't on the same scale — a 0.7 from
+e5-large and a 0.7 from bge-m3 don't mean the same thing. Reciprocal-rank fusion sidesteps that:
+it only asks "how highly did each retriever rank this candidate," which is scale-free, and uses
+that purely to decide which candidates make it into the capped set that gets fully featurized and
+scored. Once a candidate survives, the actual LightGBM/cross-encoder stack sees each retriever's
+raw score, rank, and per-query percentile as independent features, plus how many retrievers found
+it at all. This lets the model learn its own trust weighting per model (e.g. relying more on one
+embedding family for Indian transliterated names, the other for French text) rather than that
+being decided upfront by a hand-tuned blend.
+
+## 4. Data integrity
+
+Before any training or test run, `src/manifest.py` reports, for every raw file and every
+embedding source: row count, unique-ID count, duplicate IDs, ID-prefix correctness, and whether
+the embedding manifest's row count matches the data file. It also flags whether any S2/S3 ID is
+claimed as a true match by more than one S1 entity in the ground truth — if that count is ever
+nonzero, entity-based fold splitting is no longer sufficient and folds need to be built from
+connected components of the full label graph instead of individual S1 groups.
+
+## 5. Run
+
+1. Fill `PATHS` in `src/config.py` — the four raw source files, ground truth, and a work
+   directory. Parquet or TSV both work; only `entity_id, business_name, business_address,
+   country` are read.
+2. Point `CFG["emb_sources"]` at your two pre-computed embedding sets (same shard/manifest
+   format for both — `<dir>/<split>/source{1,2,3}/manifest.json` + `part-*.npy` files):
+   ```python
+   CFG["emb_sources"] = {
+       "e5":  "/path/to/e5_multilingual_large",
+       "bge": "/path/to/bge_m3",
+   }
+   ```
+   Every run verifies each source's row count and unit-norm vectors, and — on training data —
+   that true pairs score far above random pairs; on test data, that nearest neighbours usually
+   share a name word. A run stops rather than silently continuing if any source doesn't line up.
+3. Run the integrity manifest first:
+   ```bash
+   python src/manifest.py --data-dir <folder with train_source1/2/3.tsv> --split train \
+       --emb-dir e5=<e5 train dir> --emb-dir bge=<bge train dir> \
+       --ground-truth <folder>/train_ground_truth.tsv
+   ```
+   Fix anything it reports before proceeding.
+4. Train + validate:
+   ```bash
+   python src/run_train_eval.py --n-ce 20000 --n-stack 10000 --n-val 10000   # quick run
+   python src/run_train_eval.py                                              # default sizes
+   python src/run_train_eval.py --split B                                    # US -> India transfer
+   python src/run_train_eval.py --split B --train-country india --val-country us
+   ```
+   Results: `work/results_<run>/report.md` (+ `metrics.json`, `errors.tsv`, `fn_stages.tsv`,
+   `feature_importance.tsv`, `resolved_config.json`, `split_ids.json`).
+   Trained pieces: `work/models_<run>/` and `work/ce_model_<run>/`.
+5. Test inference (writes the submission files):
+   ```bash
+   python src/manifest.py --data-dir <test dir> --split test \
+       --emb-dir e5=<e5 test dir> --emb-dir bge=<bge test dir>
+   python src/run_test.py --models work/models_A --test-dir <folder with test_source1/2/3> \
+       --emb-sources e5=<e5 test dir> bge=<bge test dir> \
+       --validator ../utils/validate_submission.py --validator-test-dir <folder with test TSVs>
+   ```
+   Output: `output/matching_results.tsv` (the file scored on the leaderboard) and
+   `output/candidate_pairs.tsv` (the blocking audit trail).
+
+## 6. Rules compliance
+
+Models: `intfloat/multilingual-e5-large` (MIT, 560M) and `BAAI/bge-m3` (MIT, 568M) for
+retrieval embeddings, `microsoft/mdeberta-v3-base` (MIT, ~278M) as the cross-encoder, LightGBM
+(MIT) as the tabular scorer — all well under the 8B parameter limit. No external data,
+dictionaries, APIs, or geocoding: every suffix and abbreviation rule is learned from the provided
+training labels, and country is only ever compared for string equality.
+
+## 7. Code layout
+
+```
+src/
+├── config.py            # paths + tunable settings (fill in before running)
+├── defaults.py          # settings not in config.py; config.py values always win
+├── manifest.py          # pre-flight integrity report (run before any training/test run)
+├── data.py              # loading, ground-truth attachment, sanity checks, splits
+├── text_utils.py        # normalisation, number/alnum-token extraction, abbreviation matching
+├── transforms.py        # per-fold learned rules; rule-free vs rule-applied text views
+├── embeddings.py         # loads + aligns pre-computed embedding shards for one source
+├── blocking.py           # dense (per source) + lexical retrieval, RRF-fused candidate union
+├── features.py           # pair feature computation
+├── cross_encoder.py      # mDeBERTa fine-tuning + inference
+├── stacker.py             # LightGBM K-fold training/inference
+├── decide.py              # calibration + exact expected-F0.5 decision + one-owner arbitration
+├── evaluate.py             # macro F0.5 (matches the problem statement's worked example)
+├── run_train_eval.py       # end-to-end training + validation
+└── run_test.py             # test inference + submission file writing
+```
+
+## 8. Relationship to any other approach
+
+This pipeline is fully standalone: nothing here depends on, imports from, or assumes the
+existence of any other codebase or prior attempt at this problem. It can be read, run, and
+audited entirely on its own.

model-4/Documentation_template.md

@@ -0,0 +1,129 @@
+# ML Challenge 2026: Business Entity Resolution Solution Template
+
+**Team Name:** [Your Team Name]
+**Team Members:** [List all team members]
+**Submission Date:** [Date]
+
+---
+
+## 1. Executive Summary
+
+A two-stage blocking-and-ranking pipeline. Three independent retrievers (multilingual-e5-large, bge-m3 and an
+IDF-weighted lexical index), run per country, are fused by reciprocal rank. Two LightGBM stages score the
+candidates: a cheap stage-1 model over all fused candidates, then a stage-2 model over the top 16 that adds
+a fine-tuned mDeBERTa-v3 cross-encoder. Each S1 entity's final set is chosen from calibrated probabilities
+with one-owner arbitration. Every text rule is country-agnostic and label-free (legal forms are discovered
+from unlabeled names), so the unseen country, France, is processed exactly like US and India.
+Validation macro F0.5: **0.98298**.
+
+---
+
+## 2. Methodology
+
+### 2.1 Problem Analysis
+
+- **Scale:** train 2.21M S1 / 10.32M S2+S3; test 1.73M S1 / 9.97M S2+S3 (France: 259k S1, 1.43M S2+S3).
+- **Label structure:** 5.6% of S1 are singletons; the rest have 1–11 matches (typically 2–4). No S2/S3 record
+  belongs to two S1 entities, so a one-owner constraint is valid.
+- **Hard negatives dominate:** the same street with a neighbouring house number (`17` vs `18 R DE VALMY`),
+  and the same address with one name word swapped (`Supérieure Sante EURL` vs `Supérieure Club EURL`).
+- **Name noise:** legal forms at the end, the start or in brackets (`SARL X`, `[SAS] X`, `S.A.S.`), glued web
+  forms (`shidokanecolesarl.com`), trade names (`X trading as Y`), typos, Indian-script transliterations
+  (11% of test S2 names are Devanagari, Telugu, Tamil etc.).
+- **Address noise (France):** 40% of "Rue" appear as `R`/`R.`; `BD`, `AV`/`AVE`, `ALL`, `IMP`, `RTE`, `PL`,
+  `CH`; the region (S1) vs the department (S2/S3); `N°`, `Nº`, `#`, `Bis`, `3B`, ranges `2 - 6`; postcodes in
+  under 0.5% of records.
+
+### 2.2 Solution Strategy
+
+**Approach Type:** Blocking + two-stage classifier cascade (LightGBM + cross-encoder), calibrated
+expected-F0.5 / threshold decision with one-owner arbitration.
+**Core Innovation:** label-free, country-agnostic normalisation (legal forms discovered from unlabeled
+text using mutual exclusivity, abbreviation matching by a generic prefix/subsequence rule, segment-aware
+address parsing with house-number states), so an unseen country needs no special handling.
+
+---
+
+## 3. Candidate Generation (Blocking)
+
+- **Blocking keys used:**
+  - Dense: exact cosine top-100 per country with multilingual-e5-large and bge-m3 (GPU, no approximation).
+  - Lexical: exact IDF-weighted cosine top-100 per country over name tokens, 5-character name prefix,
+    street tokens, locality tokens, house-number+street composites, postcodes and alphanumeric unit tokens.
+  - Reciprocal-rank fusion (k0 = 60) decides which candidates survive a cap of K = 64 per S1 (smallest K
+    reaching 99.5% of the recall at K = 100). The model still sees each retriever's own score, rank and gap.
+  - A stage-1 LightGBM keeps the top 16 candidates per S1 (p ≥ 0.003). This final set is
+    `candidate_pairs.tsv`.
+- **Candidate pairs generated:** 110,882,816 test pairs after fusion; **9,980,755 test pairs (5.8 per S1)** in `candidate_pairs.tsv`.
+- **How you ensured true matches were not lost:** recall was measured on held-out train groups
+  (1.04M true pairs). Fused recall at K = 64: **98.50%** (US 99.07%, India 97.65%), above any
+  single retriever (e5 97.4%, bge 96.3%, lexical 91.1% at 100). The stage-1 filter keeps 99.92% of the
+  true pairs that blocking found.
+
+---
+
+## 4. Matching Model
+
+**Features used (~70, all on rule-free text, no country branching):**
+- Name features: ratio, Jaro-Winkler, Levenshtein, token-sort / token-set / partial ratio on the name with
+  legal forms removed; full-name ratio; no-space ratio and partial ratio (glued / web-style names);
+  abbreviation- and typo-aware token coverage in both directions; token Jaccard; counts of unmatched
+  tokens; containment (trade names); acronym match; first-token match; legal-form state (none / one missing
+  / same / compatible / conflict).
+- Address features: full-address ratio / token-set / partial / Jaccard; street-segment abbreviation-aware
+  coverage, ratio and no-space ratio; locality coverage (does each side's city or region appear in the
+  other's address); house-number state (equal / suffix differs / range contains / conflict / missing) and
+  log distance; numeric-set and postcode states; alphanumeric unit-token state.
+- Other: per-retriever score, rank and gap; RRF score and rank; retriever support count; exact e5 / bge
+  cosine with rank and gap inside the S1; S2↔S3 corroboration (max cosine to a top-10 candidate from the
+  other source, near-duplicate count); stage-1 probability, rank and gap; cross-encoder logit, rank and gap;
+  non-Latin-script and empty-address flags; candidate source.
+
+**Model type:** LightGBM (5 grouped folds by S1) at two stages, with a fine-tuned
+`microsoft/mdeberta-v3-base` cross-encoder (MIT, 278M) whose score is a stage-2 feature. The
+cross-encoder is trained on a disjoint S1 split, so its scores on the stack, validation and test sets
+are out of sample.
+**Threshold selection method:** cross-fitted isotonic calibration on stack-split out-of-fold scores.
+Exact expected-F0.5 per S1 (Poisson-binomial DP) and a threshold grid are both evaluated on the stack
+split only; the best is used. Chosen: **cross-encoder variant + exact expected-F0.5 (stack 0.98248 vs best threshold 0.98223 at t=0.70)**. The validation split is used only for reporting.
+
+---
+
+## 5. Results & Error Analysis
+
+- **F_0.5 Score (macro):** **0.98298** on 100k held-out S1 (US 0.98325, India 0.98258).
+- **Leave-one-country-out proxy:** with stage-1 features only (no cross-encoder), a model trained on one country and applied to the other loses 3-12 points: US->US 0.973 vs US->India 0.854; India->India 0.965 vs India->US 0.934. The loss is largest going into India (Indian scripts, landmark addresses unseen in US data); France, being Latin-script with structured addresses, is closer to the milder case, and the final model is trained on both countries.
+- **France (no labels):** predicted singleton rate 5.99% and mean matches 3.15, vs US
+  5.83%/3.32 and India 5.93%/3.29; nearest-neighbour name-word agreement
+  98.96% (US 98.64%, India 93.32%).
+- **Common false positives (wrong merges):** a singleton S1 linked to a near-identical business: same legal form and similar name on a nearby street or house number (e.g. `Graham Truist PC`, `Steinmann National Inc.`). Among the 2,000 lowest-scoring validation entities, 97 are singletons wrongly merged and only 10 received exclusively wrong matches; the rest are partial.
+- **Common false negatives (missed matches):** (1) true matches that retrieval never surfaced, typically very short/generic names (`Amber`) or long landmark-heavy Indian addresses. They cap recall at 98.4%. (2) Partial matches, where the model keeps the confident records and drops heavily corrupted ones (glued names, transliterated script, missing house number). 1,646 of the 2,000 worst entities are partial rather than wrong.
+
+---
+
+## 6. Conclusion
+
+A retrieval-heavy, precision-first cascade reaches 0.983 macro F0.5 on held-out US/India data. Its fused blocking recall is 98.5%, and its expected-F0.5 decision is calibrated: predicted singleton rates match the true 5.6%. Because every normalisation rule is label-free and country-agnostic, France receives exactly the same processing, and its test-time prediction profile (6.0% singletons, 3.15 matches per entity) matches the labelled countries. Lessons: exact expected-F0.5 beats a global threshold, a cross-encoder on a small top-K set gives most of the lift, and on shared hardware the pipeline had to be engineered for resumability and GPU/CPU placement as much as for accuracy.
+
+---
+
+## Appendix
+
+### A. Code Artefacts
+
+`code/business_entity_resolution/src/`: `run_pipeline.py` is the entry point, a resumable dependency-graph
+runner. `scripts/run.sh start` reproduces both output files from the raw data: normalisation → retrieval →
+fusion → features → stage 1 → cross-encoder → stage 2 → decision → `output/`. Each stage checkpoints per
+shard and a restart resumes. See `IMPLEMENTATION.md` for the stage list, GPU placement and settings.
+
+### B. Additional Results
+
+| Stage | Metric | Value |
+|---|---|---|
+| Retrieval | fused recall@64 / @100 | 98.50% / 98.78% |
+| Retrieval | e5 / bge / lexical recall@100 | 97.36% / 96.31% / 91.06% |
+| Stage 1 | AUC (stack OOF / val) | 0.99977 / 0.99978 |
+| Stage-2 set | true pairs kept (vs blocking) | 99.92% |
+| Cross-encoder | dev AUC (580k pairs, 4,529 steps) | 0.99964 |
+| Stage 2 | macro F0.5 (val, 100k S1) | 0.98298 |
+| Top features | gain | cross-encoder logit, stage-1 prob, their gaps to the S1's best, RRF rank, triangle cosine |

model-4/IMPLEMENTATION.md

@@ -0,0 +1,144 @@
+# model-4 — implementation notes and run guide
+
+`DESIGN.md` is the original design (it was the README before 2026-09-27). This file describes the pipeline **as implemented** in `src/`, how to run and resume
+it, and every place the implementation departs from the design (with the reason).
+
+## Run
+
+```bash
+cd /scratch/user4/dai/model-4
+scripts/run.sh start            # run / resume the whole pipeline, detached (survives SSH drops)
+scripts/run.sh status           # per-stage state: done / pending / FAILED xN
+scripts/run.sh logs             # orchestrator log;  scripts/run.sh tail <stage>  for one stage
+scripts/run.sh stop             # stop everything (all shards written so far are kept)
+M4_SMOKE=1 scripts/run.sh start # small end-to-end check (3k/3k/2k train S1, 5k test S1) in work_smoke/
+```
+
+Outputs: `output/matching_results.tsv` (leaderboard file) and `output/candidate_pairs.tsv`.
+Reports: `work/reports/` — `report.md` (validation F0.5), `blocking_recall.json`, `s2set_train.json`,
+`split_B.json` (leave-one-country-out), `test_report.json` (per-country prediction stats, shift flags,
+validator output), `feature_importance.tsv`, `val_errors.tsv`, `manifest.json`.
+
+Environment: `.venv` (Python 3.12) chains onto `bge-out/.venv` and adds `anyascii`; pinned versions in
+`requirements.txt`. Paths and settings live in `src/config.py`.
+
+## Fail-proofing
+
+- **Stages are separate processes** run by `src/run_pipeline.py` over a dependency graph. A stage writes
+  `work/state/<stage>.done` only on success; a restart skips finished stages.
+- **Sharded checkpoints.** Retrieval, fusion, cosine, features, stage-1 scoring and cross-encoder inference
+  write one file per 25k-query shard (dense retrieval: per shard × country). All writes are atomic
+  (temp file + rename), so a crash never leaves a half-written file that looks finished.
+- **Cross-encoder training** checkpoints model/optimizer/scheduler every 1,000 steps and resumes at the exact
+  step with the same data order. A non-finite gradient aborts the stage instead of saving a corrupted checkpoint.
+- **LightGBM** fold models are saved as they finish; a rerun loads them instead of retraining.
+- **Retries.** A failed stage is retried up to 3 times with back-off; only its dependents are blocked.
+- **Shared-GPU patience.** CUDA out-of-memory (other users filling the card) makes the stage free its cache,
+  wait 60 s and retry, for up to 3 h, instead of failing. Batch sizes halve automatically on OOM.
+- **Single runner.** A file lock prevents two orchestrators on the same work dir.
+
+## GPU placement
+
+| Stage | Device | Memory needed | Why |
+|---|---|---|---|
+| dense retrieval (e5, bge; train, test) | GPU | ~16 GB | exact cosine top-100; one country's pool on the card at a time |
+| exact cosine + S2↔S3 triangle | GPU | ~6 GB | pool stays in RAM; only each batch's candidates go to the GPU |
+| cross-encoder training | GPU | ~16 GB | mDeBERTa-v3-base, bf16 autocast, fp32 master weights |
+| cross-encoder inference (test) | **both GPUs** | ~8 GB each | shards split across every GPU with room |
+| lexical retrieval, fusion, features, LightGBM, decision | CPU | — | 24 worker processes (the server is shared) |
+
+The scheduler sends each GPU stage to the card with the most free memory (from `nvidia-smi`, minus memory
+reserved by our stages started in the last 5 minutes), runs the train and test branches concurrently,
+and waits when neither card has room.
+
+## Pipeline as built
+
+1. **manifest** — row/ID/prefix checks, embedding row-order and unit-norm checks, ground-truth coverage,
+   S2/S3 multi-owner check (0 → one-owner arbitration valid), true-vs-random pair cosine per embedding.
+2. **prep** — normalises all 24M records from the raw name/address text (the processed parquet columns are
+   not used: they leave `R`/`BD` unexpanded, miss prefix legal forms like `SCI X`, and mis-detect postcodes).
+3. **splits** — disjoint S1 groups: ce 120k, stack 200k, val 100k (random, mixed-country).
+4. **retrieval** — e5-large and bge-m3 exact top-100 per country; lexical top-100 over an IDF-weighted
+   sparse index (name tokens, 5-char name prefix, street tokens, locality tokens, house-number+street
+   composites, postcodes, alphanumeric unit tokens), per country.
+5. **fusion** — RRF (k0=60) decides survival; cap K chosen automatically as the smallest K reaching 99.5% of
+   the recall at K=100 (smoke run: K=64). Each retriever's own score/rank/gap stays a separate feature.
+6. **cosine + triangle** — exact e5/bge cosine for every candidate (including ones a retriever missed), and
+   S2↔S3 corroboration: max cosine to a top-10 candidate from the *other* source, and near-duplicate count.
+7. **features** — ~70 country-agnostic features (see below).
+8. **stage 1** — LightGBM, 5 grouped folds on the stack split (negatives beyond RRF rank 32 subsampled),
+   out-of-fold scores for stack, fold-averaged for val/test.
+9. **stage-2 set** — per S1, the top 16 stage-1 candidates with p ≥ 0.003. **This is `candidate_pairs.tsv`**:
+   the exact set the final model scores.
+10. **cross-encoder** — mDeBERTa-v3-base fine-tuned on ce-split positives + hard negatives (both text orders),
+    scored on the stage-2 set. The ce split is disjoint from stack/val, so its scores are leak-free.
+11. **stage 2** — LightGBM with and without the cross-encoder features, 5 grouped folds; cross-fitted isotonic
+    calibration; decision family (exact expected-F0.5 vs. threshold grid) chosen on the stack split only;
+    validation is reported, never used for selection.
+12. **decision** — per S1, then one-owner arbitration to a fixpoint (higher-probability claim keeps a
+    contested record; the loser re-decides without it).
+13. **output** — both TSVs, in-script checks (one row per S1, no duplicates, S2/S3 only, matches ⊆
+    candidates), the official validator with `--check-ids`, and per-country prediction statistics.
+14. **eval_B** — leave-one-country-out proxy for France (US→India, India→US), report only.
+
+## Features (all on rule-free text, no country branching)
+
+- **Retrieval:** e5/bge/lexical score, rank, gap to best; RRF score and rank; how many retrievers found it;
+  exact e5/bge cosine, its gap and rank within the S1; best cosine of the S1.
+- **Name** (legal forms removed): ratio, Jaro-Winkler, Levenshtein, token-sort, token-set, partial ratio,
+  full-name ratio; **no-space ratio / partial** (`shidokanecolesarl.com` vs `Shidokan Ecole SARL`);
+  abbreviation/typo-aware token coverage both ways; token Jaccard; unmatched-token counts (catches
+  descriptor words like `Distribution`); containment (trade-name / DBA); acronym match; first-token match;
+  legal-form state (none / one missing / same / compatible e.g. `pvt ltd`~`private limited` / conflict).
+- **Address:** full-address ratio / token-set / partial / Jaccard; **street** abbreviation-aware coverage
+  both ways, street ratio and no-space ratio; **locality coverage** — does each side's city/region appear in
+  the other's address (handles region-vs-department and reordering); **house-number state** (both missing /
+  one missing / equal / same base, different suffix `3B`≠`3` / range contains `2 - 6` / conflict) and
+  log distance; numeric-set Jaccard and state; postcode state (incl. length conflict); alphanumeric unit
+  token state (`C-199`, `248/A`).
+- **Context:** token counts, empty-address and non-Latin-script flags per side, candidate source (S2/S3),
+  candidate count of the S1.
+
+## Departures from the README design (and why)
+
+| README | Implemented | Reason |
+|---|---|---|
+| Legal-suffix / abbreviation maps learned from training positives | Legal forms discovered **label-free per country** from all names (boundary position + frequency + mutual exclusivity + neighbour-diversity); abbreviations matched by a generic prefix/subsequence rule | France has no labels. A frequency-only rule would strip descriptors (`groupe`, `distribution`) and `notre`/`pierre`; the exclusivity and neighbour tests reject them. Same treatment for every country, so nothing leaks across folds |
+| Cross-encoder probability as a feature for every candidate | **Two-stage cascade**: cheap stage-1 LightGBM on all K candidates → top-16 → cross-encoder → stage-2 LightGBM | 1.73M test S1 × 64 = 111M pairs is days of mDeBERTa on shared GPUs; the top-16 set keeps 99.7% of blocking's true matches (smoke) |
+| Cross-encoder OOF on the training side | CE trained on the disjoint **ce split**; stack/val/test scores are all out-of-sample | Same leakage guarantee, one training run instead of K |
+| Cross-encoder on all ce pairs | Capped at 600k sampled pairs, batch 128, 1 epoch | ~107 pairs/s on the shared GPU; the CE is one feature among ~70 |
+| Split B trains the full stack | Split B = stage-1-style model without the cross-encoder | A CE trained on both countries would leak the held-out country |
+| Same-country flag feature | Dropped | Retrieval is per-country, so it is always 1 |
+| e5-small embeddings | Not used | Only 8,000 smoke-run query vectors exist; e5-large + bge-m3 are complete |
+
+## Results (full run, 2026-09-26)
+
+- Validation macro F0.5 **0.98298** (US 0.98325, India 0.98258); chosen: cross-encoder variant + exact expected-F0.5.
+- Blocking recall@64 98.50%; stage-2 set keeps 99.92% of blocking's true pairs (5.2 candidates per S1).
+- Test: France 5.99% predicted singletons / 3.15 matches per S1 (US 5.83% / 3.32, India 5.93% / 3.29); no
+  country-shift flags; official validator with `--check-ids`: PASS.
+
+## GPU helpers added during the full run
+
+- `src/gpu_lgb.py` — exact LightGBM tree evaluation on the GPU (same trees and decision rule; float32
+  thresholds rounded down so `x <= t32` equals `x <= t` for float32 features). Verified bit-identical to
+  `Booster.predict` on 200k rows (max |diff| 8.9e-16).
+- `src/stage1_test_gpu.py` — fills stage-1 test shards on the GPUs from the end while the CPU stage fills
+  from the front (the CPU was overloaded by other users; this cut ~2 h). `python stage1_test_gpu.py verify`.
+- `src/ce_infer_help.py` — work-stealing helper for cross-encoder inference when one GPU is much busier.
+- Cross-encoder workers now re-check a shard before scoring it (a helper may have finished it).
+
+## Paths moved on 2026-09-26
+
+Another user of the `user4` account reorganised `/scratch/user4/dai/dataset/` during the run (the raw TSVs
+and `dataset/utils/` are gone; the parquet files are now in `dataset/og/`, same SHA-256). `config.py` now
+reads `dataset/og/`, and the validator plus its test TSVs live inside model-4 (`utils/`, `work/test_tsv/`).
+
+## Bugs found and fixed during the smoke run (for the record)
+
+- transformers 5 loads mDeBERTa in **fp16** by default → AdamW on fp16 weights gave NaN after step 1.
+  Now loaded with `dtype=torch.float32` (bf16 autocast for speed), plus a non-finite-gradient guard.
+- Lexical query matrix was built with CSR arguments in the wrong order (scores ~18,000) — fixed, re-run.
+- `N°33` did not parse as house number 33 (`°` → `" o "`); now `"o "`.
+- rapidfuzz on numpy object arrays was ~10× slower than on lists; metric used `np.isin` against 7.6M
+  ground-truth keys per call (10 s) → binary search on a cached sorted array (0.03 s).

model-4/README.md

@@ -0,0 +1,219 @@
+# model-4 — Business Entity Resolution
+
+Status on 2026-09-27: **finished**. The full training, validation and test run completed at 21:44 on
+2026-09-26. The submission in `output/` passes the organisers' validator with `--check-ids`. Every number
+below was re-read from `work/reports/`, `work/models/`, the stage logs and the output files on 2026-09-27.
+
+| | |
+|---|---|
+| Validation macro F0.5 (100,000 held-out S1, US + India) | **0.98298** (US 0.98325, India 0.98258) |
+| Chosen model | stage-2 LightGBM **with** the cross-encoder + exact expected-F0.5 decision |
+| **Leaderboard score** | **0.979241** (model-3: 0.976423) |
+| France F0.5 implied by the leaderboard (see §2) | about **0.959** (model-3: about 0.946) |
+| Submission | `output/matching_results.tsv` (95.7 MB) + `output/candidate_pairs.tsv` (151 MB) |
+| Validator (`utils/validate_submission.py --check-ids`) | **PASS** — 1,732,544 rows |
+
+Companion documents:
+- `IMPLEMENTATION.md`: every stage, the features, the fail-proofing, GPU placement, and every departure
+  from the design.
+- `DESIGN.md`: the original design this was built from.
+- `Documentation_template.md`: the filled-in challenge write-up.
+- `Colab/README_COLAB.md`: the standalone Google Colab copy.
+
+## 1. Pipeline
+
+```
+organiser records (dataset/og/*.parquet: only entity_id, business_name, business_address, country are read)
+  │  prep: own normalisation of all 24.2M records from the raw text; legal forms discovered per
+  │        country WITHOUT labels; address parsed into segments (house number, street, locality, postcode)
+  ▼
+splits: disjoint train S1 groups  ce 120k | stack 200k | val 100k   (seed 13, mixed country)
+  ▼
+retrieval, per country, exact:
+  multilingual-e5-large top-100 (pre-computed vectors)   bge-m3 top-100 (pre-computed vectors)
+  IDF lexical top-100 (name tokens, name prefix, street, locality, house+street, postcode, unit tokens)
+  ▼  reciprocal-rank fusion (k0=60) decides survival; cap K=64 chosen automatically
+  ▼  exact e5/bge cosine for every candidate + S2↔S3 "triangle" corroboration
+67 pair features ──► stage-1 LightGBM (5 grouped folds) ──► top-16 per S1 with p ≥ 0.003
+  ▼                                                           (= candidate_pairs.tsv)
+cross-encoder microsoft/mdeberta-v3-base, fine-tuned on the ce split ──► logit, rank, gap
+  ▼
+stage-2 LightGBM (73 features, 5 grouped folds) ──► cross-fitted isotonic calibration
+  ▼
+exact expected-F0.5 per S1 (Poisson-binomial DP) + one-owner arbitration to a fixpoint
+  ▼
+output/matching_results.tsv, output/candidate_pairs.tsv, validator, per-country checks
+```
+
+No rule depends on a country name. France goes through exactly the same code path as US and India.
+
+## 2. Results
+
+**Validation** (`work/reports/report.md`, `work/models/stage2_bundle.json`). Everything was selected on the
+stack split; val is used only for reporting.
+
+| Stage | Metric | Value |
+|---|---|---|
+| Retrieval | fused recall at K=64 / K=100 | 98.50% / 98.78% (US 99.07%, India 97.65% at 64) |
+| Retrieval | recall at 100, each retriever alone | e5 97.36%, bge 96.31%, lexical 91.06% |
+| Stage 1 | AUC, stack out-of-fold / val | 0.99977 / 0.99978 |
+| Stage-2 set | true pairs kept vs. blocking / vs. all true pairs | 99.92% / 98.43% |
+| Cross-encoder | held-out AUC (579,705 training pairs, 4,529 steps, 1 epoch) | 0.99964 |
+| Stage 2, no cross-encoder | stack F0.5: expected-F0.5 / best threshold (0.70) | 0.97144 / 0.97114 |
+| Stage 2, with cross-encoder | stack F0.5: expected-F0.5 / best threshold (0.70) | **0.98248** / 0.98223 |
+| **Final** | **val macro F0.5** | **0.98298** |
+
+Predicted matches per S1 on val: US 5.73% empty and 3.32 mean; India 5.87% and 3.32. Training data has
+5.58% singletons and 3.46 true matches per S1.
+
+Top stage-2 features by gain are the cross-encoder logit, the stage-1 probability, their gaps to the S1's
+best candidate, RRF rank/score and triangle cosine (`work/reports/feature_importance.tsv`).
+
+**Leave-one-country-out** (`work/reports/split_B.json`). This uses a stage-1-style model without the
+cross-encoder, because a cross-encoder trained on both countries would leak.
+
+| Train → validate | F0.5 |
+|---|---|
+| US → US | 0.973 |
+| US → India | **0.854** |
+| India → India | 0.965 |
+| India → US | 0.934 |
+
+**Test** (`work/reports/test_report.json`):
+
+| Country | S1 | No match predicted | Mean matches | Top-1 neighbour shares a name word (e5 / bge) |
+|---|---|---|---|---|
+| France | 259,452 | 5.99% | 3.15 | 98.96% / 98.75% |
+| India | 809,986 | 5.93% | 3.29 | 93.32% / 92.69% |
+| US | 663,106 | 5.83% | 3.32 | 98.64% / 98.64% |
+
+- 5,685,241 predicted pairs. 102,196 S1 are predicted singletons (5.90%). No country-shift flags.
+- `candidate_pairs.tsv` holds 9,980,755 pairs (5.8 per S1). **16,407 S1 (0.95%) have no candidate**,
+  because stage 1 gave every candidate p < 0.003. Those S1 are predicted singletons.
+- Agreement with model-3's submission (Jaccard of predicted pairs): US 0.974, India 0.971,
+  **France 0.917**. France is where the two models differ most.
+
+**Leaderboard: 0.979241.** That is 0.0037 below validation.
+- **Implied France score.** Assume US and India score on test what they scored on val (0.98325 / 0.98258).
+  Then, with the test mix (US 38.3%, India 46.8%, France 15.0%), US + India contribute 0.83570. That
+  leaves **France ≈ 0.959**.
+- **The France estimate is sensitive.** France is only 15% of the test set, so a ±0.001 error in the
+  US/India estimate moves the France figure by ∓0.006.
+- **Against model-3.** The same arithmetic gives model-3 about 0.946 on France (leaderboard 0.976423). So
+  model-4 gains about +0.0028 on the leaderboard, and most of that comes from France (+0.013 on France
+  ≈ +0.0019 on the leaderboard).
+- **France is still the main gap.** It is about 0.024 below US/India, and it accounts for about 0.0036 of
+  the 0.0037 drop from validation to the leaderboard.
+
+## 3. Run
+
+```bash
+cd /scratch/user4/dai/model-4
+scripts/run.sh start            # run / resume everything, detached (survives SSH drops)
+scripts/run.sh status           # per-stage state: done / pending / FAILED xN
+scripts/run.sh logs             # orchestrator log;  scripts/run.sh tail <stage>  for one stage
+scripts/run.sh stop             # stop (finished shards are kept)
+M4_SMOKE=1 scripts/run.sh start # small end-to-end check in work_smoke/ + output_smoke/
+.venv/bin/python src/run_pipeline.py --reset <stage>   # force a stage and its dependents to rerun
+```
+
+- Every stage is a separate process and writes `work/state/<stage>.done` on success, so a restart skips
+  finished stages.
+- Retrieval, features, stage-1 scoring and cross-encoder inference checkpoint per 25k-query shard, with
+  atomic writes.
+- Cross-encoder training checkpoints every 1,000 steps. LightGBM fold models are reused.
+- On a CUDA out-of-memory error, a stage waits and retries instead of failing. Batch sizes halve
+  automatically.
+- The scheduler puts each GPU stage on the card with the most free memory.
+- Environment overrides: `M4_GPUS=0,1`, `M4_WORK`, `M4_OUTPUT`, `M4_SMOKE`. Paths and settings are in
+  `src/config.py`.
+
+**Time.** The full run took about 6 h on the shared server (15:44–21:44, 2026-09-26), including
+CPU-contention waits.
+
+**Rerunning the final decision only** takes about 7 min and needs no GPU:
+`src/run_pipeline.py --reset stage2_test`.
+
+## 4. Layout
+
+```
+src/config.py            paths (dataset/og, embeddings, validator) + all settings (CFG)
+src/run_pipeline.py      DAG orchestrator: retries, locking, GPU scheduling, --status/--reset/--only
+src/stages.py            stage registry (one subprocess per stage)
+src/common.py            logging, atomic saves, shard paths, progress, GPU-OOM patience
+src/prep.py              manifest checks, normalisation of all records, splits
+src/text_utils.py        normalisation, label-free legal-form discovery, address parsing, abbreviation match
+src/embeddings.py        pre-computed e5 / bge vectors (memory-mapped, row-order checked)
+src/retrieve_dense.py    exact dense top-100 per country + exact cosine / triangle features
+src/retrieve_lex.py      IDF sparse lexical top-100 per country
+src/blocking.py          RRF fusion, recall probe, automatic K
+src/features.py          pair features (rapidfuzz cpdist + custom)
+src/stacker.py           stage-1 / stage-2 LightGBM, stage-2 set, submission writing + checks, split B
+src/cross_encoder.py     mDeBERTa-v3 fine-tuning + multi-GPU inference
+src/decide.py            macro F0.5, cross-fitted isotonic, exact expected-F0.5 DP, arbitration
+src/gpu_lgb.py           exact GPU evaluation of LightGBM trees (bit-identical to Booster.predict)
+src/stage1_test_gpu.py   GPU helper that fills stage-1 test shards from the end
+src/ce_infer_help.py     work-stealing helper for cross-encoder inference
+scripts/run.sh           start | status | stop | logs | tail
+utils/validate_submission.py   the organisers' validator (local copy)
+Colab/                   standalone Hugging Face-based copy of this pipeline for Google Colab
+```
+
+| Artefact | Size | What |
+|---|---|---|
+| `work/prep/` | 4.5 GB | normalised records (shared by smoke and full runs) |
+| `work/models/` | 273 MB | 5 stage-1 folds, 10 stage-2 folds (with/without CE), Split B folds, `stage2_bundle.json`, `stage2_iso.pkl` |
+| `work/ce/final/` | 1.1 GB | fine-tuned mDeBERTa cross-encoder |
+| `work/reports/` | — | `manifest.json`, `splits.json`, `blocking_recall.json`, `s2set_train.json`, `ce_train.json`, `split_B.json`, `test_report.json`, `report.md`, `feature_importance.tsv`, `val_errors.tsv`, `nn_check_*_test.json` |
+| `work/test_tsv/` | — | test TSVs written from the parquet, for the validator |
+| `work/` total | 29 GB | |
+
+The `*.fail` markers in `work/state/` (`ce_infer_test`, `eval_B`, `stage2_test`) are from earlier failed
+attempts. Each of those stages later succeeded and has a `.done` marker.
+
+## 5. Inputs
+
+| Input | Location |
+|---|---|
+| Records + ground truth | `/scratch/user4/dai/dataset/og/{train,test}_p/` (Hugging Face `Jyo-K/PopTards_Machine_Code`) |
+| e5-large vectors | `/scratch/user4/dai/hybrid_bridge/prepared{,_test}/hub/e5_multilingual_large/` |
+| bge-m3 vectors | `/scratch/user4/dai/bge-out/{train,test}/` |
+| Python | `.venv/` (Python 3.12). It chains onto `bge-out/.venv`, which points at another user's venv under `/home/user4/Jyo/`. Pinned versions are in `requirements.txt` |
+
+The manifest stage checks all of these: row counts, ID prefixes, duplicate IDs, and embedding row order
+and unit norm. It also compares true-pair vs random-pair cosine: e5 0.923 vs 0.784, bge 0.849 vs 0.418.
+The ground truth has 0 records claimed by two S1, which makes one-owner arbitration valid.
+
+## 6. Known limitations
+
+1. **France is the weak country.** The leaderboard implies about 0.959 vs about 0.983 on US/India (§2).
+   There are no France labels, so the causes can't be measured directly. The evidence:
+   - Its predicted singleton rate and match counts look like US/India.
+   - The leave-one-country-out loss is 3–12 points.
+   - France is where model-4 and model-3 disagree most (Jaccard 0.917).
+2. **The lexical index drops tokens with document frequency above 5,000 per country** (`lex_max_df`).
+   - That removes about 1,000 of the most frequent tokens in each country, such as common street and
+     legal words.
+   - model-3 found that this setting badly hurt its lexical channel. Here lexical recall at 100 is 91.1%
+     and it is one of three fused retrievers.
+   - The effect on fused recall has not been measured.
+3. **India retrieval is the weakest slice**: 97.65% recall at K=64 vs US 99.07%.
+4. **The cross-encoder is capped** at 600k training pairs and 1 epoch. More training is untested.
+5. **Other experiments read this folder's intermediates.**
+   - `/scratch/user4/dai/model-4-dual-ce/work/` symlinks `prep`, `feat`, `s2`, `ce_scores`, `splits.npz`
+     and `reports` from `model-4/work/`.
+   - Do not delete or reset those while that experiment is in use.
+6. **Shared server.** Other users share the `user4` account, both GPUs and the CPUs. The pipeline waits
+   rather than failing, so run times vary.
+
+## 7. Compliance
+
+- **Models:**
+  - `intfloat/multilingual-e5-large` (MIT, 560M)
+  - `BAAI/bge-m3` (MIT, 568M)
+  - `microsoft/mdeberta-v3-base` (MIT, 278M)
+  - LightGBM (MIT)
+  - All are far below the 8B limit.
+- No external data, dictionaries, APIs or geocoding. Legal forms and abbreviations are discovered from
+  the provided text.
+- Country is used only to partition retrieval by string equality and is never a model feature.

model-4/hf_cache/.agent_harnesses.json

@@ -0,0 +1 @@
+{"standardEnvVars": ["AI_AGENT", "AGENT"], "harnesses": {"antigravity": {"prettyLabel": "Antigravity", "docsUrl": "https://antigravity.google", "description": "Agentic development platform from Google built around Gemini.", "envVars": {"ANTIGRAVITY_AGENT": "*"}}, "augment-cli": {"prettyLabel": "Augment CLI", "repoUrl": "https://github.com/augmentcode/auggie", "docsUrl": "https://www.augmentcode.com", "description": "Auggie, the command-line coding agent from Augment Code.", "envVars": {"AUGMENT_AGENT": "*"}}, "cline": {"prettyLabel": "Cline", "repoUrl": "https://github.com/cline/cline", "docsUrl": "https://cline.bot", "description": "Open-source autonomous coding agent for VS Code.", "envVars": {"CLINE_ACTIVE": "*"}}, "cowork": {"prettyLabel": "Cowork", "docsUrl": "https://claude.com/product/cowork", "description": "Anthropic's agent for autonomous knowledge work, built on top of Claude Code.", "envVars": {"CLAUDE_CODE_IS_COWORK": "*"}}, "claude-code": {"prettyLabel": "Claude Code", "repoUrl": "https://github.com/anthropics/claude-code", "docsUrl": "https://code.claude.com/docs", "description": "Anthropic's agentic coding tool that lives in your terminal.", "envVars": {"CLAUDECODE": "*", "CLAUDE_CODE": "*"}}, "codex": {"prettyLabel": "Codex", "repoUrl": "https://github.com/openai/codex", "docsUrl": "https://developers.openai.com/codex", "description": "OpenAI's lightweight coding agent that runs in your terminal.", "envVars": {"CODEX_SANDBOX": "*", "CODEX_CI": "*", "CODEX_THREAD_ID": "*"}}, "crush": {"prettyLabel": "Crush", "repoUrl": "https://github.com/charmbracelet/crush", "docsUrl": "https://github.com/charmbracelet/crush", "description": "Charm's open-source AI coding agent for the terminal.", "envVars": {"CRUSH": "*"}}, "gemini-cli": {"prettyLabel": "Gemini CLI", "repoUrl": "https://github.com/google-gemini/gemini-cli", "docsUrl": "https://geminicli.com", "description": "Google's open-source terminal AI coding agent powered by Gemini models.", "envVars": {"GEMINI_CLI": "*"}}, "github-copilot": {"prettyLabel": "GitHub Copilot", "docsUrl": "https://docs.github.com/copilot", "description": "GitHub's AI coding assistant.", "envVars": {"COPILOT_MODEL": "*", "COPILOT_ALLOW_ALL": "*", "COPILOT_GITHUB_TOKEN": "*"}}, "goose": {"prettyLabel": "Goose", "repoUrl": "https://github.com/aaif-goose/goose", "docsUrl": "https://goose-docs.ai/", "description": "Open-source, extensible AI agent, originally from Block and now part of the Agentic AI Foundation.", "envVars": {"GOOSE_TERMINAL": "*"}}, "hermes-agent": {"prettyLabel": "Hermes Agent", "repoUrl": "https://github.com/NousResearch/hermes-agent", "docsUrl": "https://hermes-agent.nousresearch.com/docs", "description": "Nous Research's self-improving, multi-provider terminal AI agent.", "envVars": {"HERMES_SESSION_ID": "*"}}, "hi": {"prettyLabel": "hi", "repoUrl": "https://github.com/PipeNetwork/hi", "docsUrl": "https://github.com/PipeNetwork/hi#readme", "description": "Rust terminal coding agent with verification-in-the-loop."}, "kilo-code": {"prettyLabel": "Kilo Code", "repoUrl": "https://github.com/Kilo-Org/kilocode", "docsUrl": "https://kilocode.ai/docs", "description": "Open-source agentic coding agent for VS Code, JetBrains, and the terminal.", "envVars": {"KILOCODE_FEATURE": "*"}}, "kiro": {"prettyLabel": "Kiro", "docsUrl": "https://kiro.dev", "description": "AWS's agentic IDE for spec-driven AI software development.", "envVars": {"AGENT_CONTEXT_OUT": "*"}}, "openclaw": {"prettyLabel": "OpenClaw", "repoUrl": "https://github.com/openclaw/openclaw", "docsUrl": "https://openclaw.ai", "description": "Open-source, self-hosted personal AI assistant that runs on your own devices.", "envVars": {"OPENCLAW_SHELL": "*"}}, "sandbase-harness": {"prettyLabel": "SandBase Harness", "repoUrl": "https://github.com/sandbaseai/sandbase-harness", "docsUrl": "https://github.com/sandbaseai/sandbase-harness/blob/main/docs/installation.md", "description": "Local-first, self-hosted runtime for persistent AI agent sessions and MCP tools."}, "opencode": {"prettyLabel": "opencode", "repoUrl": "https://github.com/anomalyco/opencode", "docsUrl": "https://opencode.ai", "description": "Open-source AI coding agent built for the terminal.", "envVars": {"OPENCODE_CLIENT": "*"}}, "pi": {"prettyLabel": "Pi", "repoUrl": "https://github.com/earendil-works/pi", "docsUrl": "https://pi.dev", "description": "Minimal, self-extensible terminal coding agent with a unified multi-provider LLM API.", "envVars": {"PI_CODING_AGENT": "*"}}, "replit": {"prettyLabel": "Replit", "docsUrl": "https://replit.com", "description": "Cloud development environment with an AI coding agent.", "envVars": {"REPL_ID": "*"}}, "trae": {"prettyLabel": "Trae", "docsUrl": "https://trae.ai", "description": "AI-powered IDE from ByteDance.", "envVars": {"TRAE_AI_SHELL_ID": "*"}}, "vtcode": {"prettyLabel": "VTCode", "repoUrl": "https://github.com/vinhnx/VTCode", "description": "Rust coding agent with multi-provider LLM support, OS-native sandboxing, and extensible skills", "envVars": {"VTCODE": "1"}}, "warp": {"prettyLabel": "Warp", "repoUrl": "https://github.com/warpdotdev/Warp", "docsUrl": "https://docs.warp.dev", "description": "AI-powered terminal with an agentic Agent Mode.", "envVars": {"TERM_PROGRAM": "WarpTerminal"}}, "zed": {"prettyLabel": "Zed", "repoUrl": "https://github.com/zed-industries/zed", "docsUrl": "https://zed.dev", "description": "High-performance code editor with an integrated AI agent panel and terminal.", "envVars": {"ZED_TERM": "*"}}, "cursor-cli": {"prettyLabel": "Cursor CLI", "docsUrl": "https://cursor.com/docs/cli/overview", "description": "Cursor's coding agent for the command line.", "envVars": {"CURSOR_AGENT": "*"}}, "cursor": {"prettyLabel": "Cursor", "docsUrl": "https://cursor.com", "description": "AI-powered code editor.", "envVars": {"CURSOR_TRACE_ID": "*"}}, "devin": {"prettyLabel": "Devin", "docsUrl": "https://devin.ai", "description": "Autonomous AI software engineer from Cognition."}}}

model-4/hf_cache/hub/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by huggingface_hub.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/

model-4/hf_cache/hub/blobs/.huggingface-shared-blobs

@@ -0,0 +1 @@
+1

model-4/hf_cache/hub/blobs/7c/7ccdaee489cbb709d4624beb0f0556fc5552e6759bde1d881fca5c7a9ee5462c

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959
+size 1332766994

model-4/hf_cache/hub/blobs/7c/7ccdaee489cbb709d4624beb0f0556fc5552e6759bde1d881fca5c7a9ee5462c.refs

@@ -0,0 +1 @@
+models--microsoft--mdeberta-v3-base/blobs/4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959

model-4/hf_cache/hub/blobs/99/994dbaac4eaaaf1033ce39ecedd612e77b89dbfe8efd8de25ca24bea40caf6bc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead
+size 1332809049

model-4/hf_cache/hub/blobs/99/994dbaac4eaaaf1033ce39ecedd612e77b89dbfe8efd8de25ca24bea40caf6bc.refs

@@ -0,0 +1 @@
+models--microsoft--mdeberta-v3-base/blobs/6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead

model-4/hf_cache/hub/blobs/c4/c4b914ec5ab295fbee3b0e0fe62f22febb3c09d34dd3778745e0de034a76df33

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3
+size 4305025

model-4/hf_cache/hub/blobs/c4/c4b914ec5ab295fbee3b0e0fe62f22febb3c09d34dd3778745e0de034a76df33.refs

@@ -0,0 +1 @@
+models--microsoft--mdeberta-v3-base/blobs/13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3
+size 4305025

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/27b905d788d20826414d27ce9fa9765623e38ea7

@@ -0,0 +1,22 @@
+{
+		"model_type": "deberta-v2",
+		"attention_probs_dropout_prob": 0.1,
+		"hidden_act": "gelu",
+		"hidden_dropout_prob": 0.1,
+		"hidden_size": 768,
+		"initializer_range": 0.02,
+		"intermediate_size": 3072,
+		"max_position_embeddings": 512,
+		"relative_attention": true,
+		"position_buckets": 256,
+		"norm_rel_ebd": "layer_norm",
+		"share_att_key": true,
+		"pos_att_type": "p2c|c2p",
+		"layer_norm_eps": 1e-7,
+		"max_relative_positions": -1,
+		"position_biased_input": false,
+		"num_attention_heads": 12,
+		"num_hidden_layers": 12,
+		"type_vocab_size": 0,
+		"vocab_size": 251000
+}

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959
+size 1332766994

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead
+size 1332809049

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/blobs/acfd94e399c5659e4bed75f91b4ee24b111fc7a6

@@ -0,0 +1,4 @@
+{
+  "do_lower_case": false,
+  "vocab_type": "spm"
+}

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/refs/main

@@ -0,0 +1 @@
+a0484667b22365f84929a935b5e50a51f71f159d

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/refs/refs/pr/5

@@ -0,0 +1 @@
+faf660b811036d7b6dfc53aacfe5281447d4e2c6

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/snapshots/a0484667b22365f84929a935b5e50a51f71f159d/config.json

@@ -0,0 +1,22 @@
+{
+		"model_type": "deberta-v2",
+		"attention_probs_dropout_prob": 0.1,
+		"hidden_act": "gelu",
+		"hidden_dropout_prob": 0.1,
+		"hidden_size": 768,
+		"initializer_range": 0.02,
+		"intermediate_size": 3072,
+		"max_position_embeddings": 512,
+		"relative_attention": true,
+		"position_buckets": 256,
+		"norm_rel_ebd": "layer_norm",
+		"share_att_key": true,
+		"pos_att_type": "p2c|c2p",
+		"layer_norm_eps": 1e-7,
+		"max_relative_positions": -1,
+		"position_biased_input": false,
+		"num_attention_heads": 12,
+		"num_hidden_layers": 12,
+		"type_vocab_size": 0,
+		"vocab_size": 251000
+}

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/snapshots/a0484667b22365f84929a935b5e50a51f71f159d/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f89419baf0f1aaad5cab7d53901e36a8c1af8f6b4ab58b15db9af32df656ead
+size 1332809049

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/snapshots/a0484667b22365f84929a935b5e50a51f71f159d/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3
+size 4305025

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/snapshots/a0484667b22365f84929a935b5e50a51f71f159d/tokenizer_config.json

@@ -0,0 +1,4 @@
+{
+  "do_lower_case": false,
+  "vocab_type": "spm"
+}

model-4/hf_cache/hub/models--microsoft--mdeberta-v3-base/snapshots/faf660b811036d7b6dfc53aacfe5281447d4e2c6/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4809ba6a0689d9e09169f42c33531333ab5bb5eae501e26a29348ce62bea5959
+size 1332766994

model-4/output/candidate_pairs.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26ec998fe938187230d01f47ce729341d9c11bd04949b91ccfb974311c7c9762
+size 150987715

model-4/output/matching_results.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81fd20e45c80d16fffccad360e383dff0e82e65f76e9e39646792f53035ed35d
+size 95709741

model-4/requirements.txt

@@ -0,0 +1,16 @@
+# Python 3.12, CUDA 13 (torch cu130 wheel)
+anyascii==0.3.2
+huggingface_hub==1.33.0
+lightgbm==4.7.0
+numpy==2.4.4
+pandas==3.0.2
+protobuf==7.36.2
+pyarrow==25.0.1
+RapidFuzz==3.14.6
+safetensors==0.8.0
+scikit-learn==1.8.0
+scipy==1.18.1
+sentencepiece==0.2.2
+tokenizers==0.23.2
+torch==2.14.0
+transformers==5.17.0

model-4/scripts/run.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Usage: scripts/run.sh start|status|stop|logs|tail <stage>   (prefix with M4_SMOKE=1 for the smoke run)
+# The runner is detached (setsid, own process group) so it survives SSH disconnects; "start" again resumes.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+PY="$PWD/.venv/bin/python"
+W=work; [ "${M4_SMOKE:-0}" = "1" ] && W=work_smoke
+mkdir -p "$W/logs" "$W/state"
+PIDF="$W/state/pipeline.pid"
+alive() { [ -f "$PIDF" ] && kill -0 "$(cut -d' ' -f1 "$PIDF")" 2>/dev/null; }
+case "${1:-}" in
+  start)
+    if alive; then echo "already running (pid $(cut -d' ' -f1 "$PIDF"))"; exit 0; fi
+    setsid -f "$PY" -u src/run_pipeline.py "${@:2}" >> "$W/logs/pipeline.out" 2>&1 < /dev/null
+    sleep 3; alive && echo "started pid $(cut -d' ' -f1 "$PIDF")" || { echo "failed to start"; tail -5 "$W/logs/pipeline.out"; } ;;
+  status) alive && echo "RUNNING" || echo "not running"; (cd src && "$PY" run_pipeline.py --status) ;;
+  stop) alive && kill -- -"$(cut -d' ' -f2 "$PIDF")" && echo stopped || echo "not running" ;;
+  logs) tail -n "${2:-40}" "$W/logs/pipeline.out" ;;
+  tail) tail -n "${3:-40}" "$W/logs/$2.log" ;;
+  *) echo "usage: $0 start|status|stop|logs|tail <stage>"; exit 1 ;;
+esac

model-4/src/blocking.py

@@ -0,0 +1,105 @@
+"""CPU stage: reciprocal-rank fusion of e5 / bge / lexical candidates. RRF only decides which candidates survive
+the cap; every retriever's own score/rank/gap is kept as a separate feature. Train side also runs the
+recall-vs-K probe that picks the cap."""
+from multiprocessing import Pool
+
+import numpy as np
+import pandas as pd
+
+from common import Progress, get_logger, n_workers, save_json, save_parquet, shard_path, load_json
+from config import CFG, EMB_MODELS, WORK
+from decide import in_sorted
+from prep import load_prep, load_splits
+from retrieve_dense import n_shards, queries
+
+log = get_logger("block")
+RETR = list(EMB_MODELS) + ["lex"]
+G = {}
+
+
+def _load(r, split, i):
+    p = shard_path("lex", split, i, "npz") if r == "lex" else shard_path("dense", split, i, "npz", r)
+    z = np.load(p)
+    return z["cand"], z["score"].astype(np.float32)
+
+
+def _block_shard(i):
+    split, Q = G["split"], G["Q"]
+    lo = i * CFG["query_chunk"]
+    frames = []
+    for r in RETR:
+        cand, score = _load(r, split, i)
+        nq, K = cand.shape
+        f = pd.DataFrame({"qi": np.repeat(np.arange(nq), K), "c": cand.ravel(),
+                          f"{r}_r": np.tile(np.arange(K), nq), f"{r}_s": score.ravel()})
+        frames.append(f[f.c >= 0])
+    df = frames[0]
+    for f in frames[1:]:
+        df = df.merge(f, on=["qi", "c"], how="outer")
+    k0 = CFG["rrf_k0"]
+    df["rrf"] = sum((1.0 / (k0 + 1 + df[f"{r}_r"])).fillna(0) for r in RETR)
+    df["support"] = sum(df[f"{r}_r"].notna().astype(np.int8) for r in RETR).astype(np.int8)
+    df = df.sort_values(["qi", "rrf"], ascending=[True, False], kind="stable")
+    df["rrf_rank"] = df.groupby("qi").cumcount().astype(np.int16)
+    df = df[df.rrf_rank < CFG["k_max"]]
+    for r in RETR:
+        df[f"{r}_gap"] = df.groupby("qi")[f"{r}_s"].transform("max") - df[f"{r}_s"]
+        df[f"{r}_r"] = df[f"{r}_r"].fillna(CFG["dense_k"]).astype(np.int16)
+    df["q"] = Q[lo + df.qi.values].astype(np.int32)
+    df["c"] = df.c.astype(np.int32)
+    if split == "train":
+        key = df.q.values.astype(np.int64) * G["mult"] + df.c.values
+        df["y"] = in_sorted(key, G["gt_key"]).astype(np.int8)
+    else:
+        df["y"] = np.int8(-1)
+    cols = ["q", "c", "rrf", "rrf_rank", "support", "y"] + [f"{r}_{x}" for r in RETR for x in ("r", "s", "gap")]
+    for c in cols:
+        if df[c].dtype == np.float64:
+            df[c] = df[c].astype(np.float32)
+    save_parquet(df[cols].reset_index(drop=True), shard_path("block", split, i))
+    return i
+
+
+def run_block(split):
+    ns = n_shards(split)
+    Q = queries(split)
+    G.update(split=split, Q=Q)
+    if split == "train":
+        sp = load_splits()
+        mult = np.int64(1 << 32)
+        G["mult"] = mult
+        G["gt_key"] = np.sort(sp["gt_q"] * mult + sp["gt_c"])
+    todo = [i for i in range(ns) if not shard_path("block", split, i).exists()]
+    if todo:
+        prog = Progress(log, len(todo), f"block {split} shards")
+        with Pool(min(n_workers(), 16)) as pool:
+            for _ in pool.imap_unordered(_block_shard, todo):
+                prog.step()
+    if split == "train":
+        recall_probe(ns)
+
+
+def recall_probe(ns):
+    sp = load_splits()
+    n_true = sp["n_true"]
+    evalq = np.concatenate([sp["g_stack"], sp["g_val"]])
+    country = load_prep("train", "s1", ["country"]).country.values
+    df = pd.concat([pd.read_parquet(shard_path("block", "train", i),
+                                    columns=["q", "rrf_rank", "y"] + [f"{r}_r" for r in RETR]) for i in range(ns)])
+    df = df[np.isin(df.q.values, evalq) & (df.y == 1)]
+    tot = int(n_true[evalq].sum())
+    ctry = pd.Series(country[evalq]).value_counts()
+    tot_c = pd.Series(n_true[evalq]).groupby(country[evalq]).sum()
+    rep = {"true_pairs": tot, "recall_at_K": {}, "recall_at_K_by_country": {}, "per_retriever_recall@100": {}}
+    for K in CFG["k_grid"]:
+        m = df.rrf_rank < K
+        rep["recall_at_K"][K] = float(m.sum() / tot)
+        rep["recall_at_K_by_country"][K] = (df[m].groupby(country[df[m].q.values]).size() / tot_c).round(4).to_dict()
+    for r in RETR:
+        rep["per_retriever_recall@100"][r] = float((df[f"{r}_r"] < CFG["dense_k"]).sum() / tot)
+    full = rep["recall_at_K"][CFG["k_max"]]
+    K = next(k for k in CFG["k_grid"] if rep["recall_at_K"][k] >= CFG["k_recall_frac"] * full)
+    rep["chosen_K"] = K
+    save_json(rep, WORK / "reports" / "blocking_recall.json")
+    save_json({"K": int(K)}, WORK / "reports" / "k_choice.json")
+    log.info(f"blocking recall: {rep}")

model-4/src/ce_infer_help.py

@@ -0,0 +1,38 @@
+"""Work-stealing helper for cross-encoder inference: scores the shards of another rank from the END backwards
+(the owner goes forwards; shards are written atomically and both skip finished ones, so they meet).
+
+  CUDA_VISIBLE_DEVICES=<fast gpu> python ce_infer_help.py <split> <rank_to_help> <world>
+"""
+import sys
+
+from common import shard_path
+from cross_encoder import CE, _n_ce_shards, _score, _texts, _tok_model, gpu_patient, log
+
+import pandas as pd
+import torch
+from config import CFG, WORK
+
+
+def main():
+    split, rank, world = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
+    AutoTokenizer, AutoModel = _tok_model()
+    n, ns = _n_ce_shards(split)
+    s2 = pd.read_parquet(WORK / "s2" / f"{split}.parquet", columns=["q", "c"])
+    ta, tb = _texts(split, "s1"), _texts(split, "pool")
+    tok = AutoTokenizer.from_pretrained(CE / "final")
+    model = AutoModel.from_pretrained(CE / "final", dtype=torch.float32)
+    model = gpu_patient(lambda: model.cuda().eval(), log, "load cross-encoder")
+    from common import save_npy
+    for i in range(ns - 1, -1, -1):
+        if i % world != rank or shard_path("ce_scores", split, i, "npy").exists():
+            continue
+        sl = slice(i * CFG["ce_shard_pairs"], (i + 1) * CFG["ce_shard_pairs"])
+        q, c = s2.q.values[sl], s2.c.values[sl]
+        scores = _score(model, tok, ta[q], tb[c])
+        if not shard_path("ce_scores", split, i, "npy").exists():
+            save_npy(shard_path("ce_scores", split, i, "npy"), scores)
+        log.info(f"helper scored shard {i}")
+
+
+if __name__ == "__main__":
+    main()