"""Conservative text cleaning and canonical identifiers.""" from html.parser import HTMLParser import re from .profile import cluster_key, normalize_text, split_for_cluster class VisibleText(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.parts = [] self.hidden = [] def handle_starttag(self, tag, attrs): if tag in {"script", "style", "noscript"}: self.hidden.append(tag) elif not self.hidden: self.parts.append(" ") def handle_endtag(self, tag): if self.hidden: if tag == self.hidden[-1]: self.hidden.pop() else: self.parts.append(" ") def handle_data(self, data): if not self.hidden: self.parts.append(data) def clean_text(value): if not isinstance(value, str): return "" # Do not parse ordinary model-number comparisons as HTML. if re.search(r"]*>", value): parser = VisibleText() parser.feed(value) value = "".join(parser.parts) value = re.sub(r'(?<=")@[a-zA-Z]{2,3}(?:-[a-zA-Z0-9]+)*\b', "", value) return normalize_text(value) def canonical_gtins(identifiers): """Only valid GTIN checksums; seller-local SKUs are never global IDs.""" result = set() if not isinstance(identifiers, list): return [] for entry in identifiers: if not isinstance(entry, dict): continue for kind, raw in entry.items(): if kind.lower().lstrip("/") not in {"gtin", "gtin8", "gtin12", "gtin13", "gtin14", "ean", "upc"}: continue for code in re.findall(r"(?