File size: 2,019 Bytes
1cd137c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
from __future__ import annotations

from difflib import SequenceMatcher

from reference_db.phase_3.grouping.models import CandidatePair, ProductRecord


def normalize_title(title: str) -> str:
    return " ".join(str(title).lower().split())


def calculate_similarity(title_a: str, title_b: str) -> float:
    normalized_a = normalize_title(title_a)
    normalized_b = normalize_title(title_b)
    return SequenceMatcher(None, normalized_a, normalized_b).ratio()


def generate_candidate(
    product_a: ProductRecord,
    product_b: ProductRecord,
    threshold: float = 0.85,
) -> CandidatePair | None:
    similarity = calculate_similarity(product_a.core_title, product_b.core_title)

    if similarity < threshold:
        return None

    return CandidatePair(
        external_id_a=product_a.external_id,
        external_id_b=product_b.external_id,
        product_name_a=product_a.product_name,
        product_name_b=product_b.product_name,
        similarity=similarity,
    )


def build_block_key(title: str) -> str:
    normalized = normalize_title(title)
    tokens = [token for token in normalized.split() if not token.isdigit()]
    return " ".join(tokens[:4])


def generate_candidates(
    products: list[ProductRecord],
    threshold: float = 0.85,
) -> list[CandidatePair]:
    blocks: dict[str, list[ProductRecord]] = {}

    for product in products:
        block_key = build_block_key(product.core_title)

        if not block_key:
            continue

        blocks.setdefault(block_key, []).append(product)

    candidates: list[CandidatePair] = []

    for block_products in blocks.values():
        for index, product_a in enumerate(block_products):
            for product_b in block_products[index + 1:]:
                candidate = generate_candidate(
                    product_a,
                    product_b,
                    threshold=threshold,
                )

                if candidate is not None:
                    candidates.append(candidate)

    return candidates