Download src/reference_db/phase_3/grouping/candidates.py from offCanada/reference-db-system: direct link, hf CLI and curl.
- Browser
- Download file 2.02 kB
-
https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_3/grouping/candidates.py
- Command line
-
hf download hf://offCanada/reference-db-system/src/reference_db/phase_3/grouping/candidates.py
-
curl -L -o candidates.py https://huggingface.co/offCanada/reference-db-system/resolve/main/src/reference_db/phase_3/grouping/candidates.py
2.02 kB
| from __future__ import annotations | |
| from difflib import SequenceMatcher | |
| from reference_db.phase_3.grouping.models import CandidatePair, ProductRecord | |
| def normalize_title(title: str) -> str: | |
| return " ".join(str(title).lower().split()) | |
| def calculate_similarity(title_a: str, title_b: str) -> float: | |
| normalized_a = normalize_title(title_a) | |
| normalized_b = normalize_title(title_b) | |
| return SequenceMatcher(None, normalized_a, normalized_b).ratio() | |
| def generate_candidate( | |
| product_a: ProductRecord, | |
| product_b: ProductRecord, | |
| threshold: float = 0.85, | |
| ) -> CandidatePair | None: | |
| similarity = calculate_similarity(product_a.core_title, product_b.core_title) | |
| if similarity < threshold: | |
| return None | |
| return CandidatePair( | |
| external_id_a=product_a.external_id, | |
| external_id_b=product_b.external_id, | |
| product_name_a=product_a.product_name, | |
| product_name_b=product_b.product_name, | |
| similarity=similarity, | |
| ) | |
| def build_block_key(title: str) -> str: | |
| normalized = normalize_title(title) | |
| tokens = [token for token in normalized.split() if not token.isdigit()] | |
| return " ".join(tokens[:4]) | |
| def generate_candidates( | |
| products: list[ProductRecord], | |
| threshold: float = 0.85, | |
| ) -> list[CandidatePair]: | |
| blocks: dict[str, list[ProductRecord]] = {} | |
| for product in products: | |
| block_key = build_block_key(product.core_title) | |
| if not block_key: | |
| continue | |
| blocks.setdefault(block_key, []).append(product) | |
| candidates: list[CandidatePair] = [] | |
| for block_products in blocks.values(): | |
| for index, product_a in enumerate(block_products): | |
| for product_b in block_products[index + 1:]: | |
| candidate = generate_candidate( | |
| product_a, | |
| product_b, | |
| threshold=threshold, | |
| ) | |
| if candidate is not None: | |
| candidates.append(candidate) | |
| return candidates |