beno-rohman's picture
Publish WDC product embedding checkpoint, training source, and evaluation
a360597 verified
Raw History Blame Contribute Delete
3.37 kB
"""Conservative text cleaning and canonical identifiers."""
from html.parser import HTMLParser
import re
from .profile import cluster_key, normalize_text, split_for_cluster
class VisibleText(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts = []
self.hidden = []
def handle_starttag(self, tag, attrs):
if tag in {"script", "style", "noscript"}:
self.hidden.append(tag)
elif not self.hidden:
self.parts.append(" ")
def handle_endtag(self, tag):
if self.hidden:
if tag == self.hidden[-1]:
self.hidden.pop()
else:
self.parts.append(" ")
def handle_data(self, data):
if not self.hidden:
self.parts.append(data)
def clean_text(value):
if not isinstance(value, str):
return ""
# Do not parse ordinary model-number comparisons as HTML.
if re.search(r"</?[a-zA-Z][^>]*>", value):
parser = VisibleText()
parser.feed(value)
value = "".join(parser.parts)
value = re.sub(r'(?<=")@[a-zA-Z]{2,3}(?:-[a-zA-Z0-9]+)*\b', "", value)
return normalize_text(value)
def canonical_gtins(identifiers):
"""Only valid GTIN checksums; seller-local SKUs are never global IDs."""
result = set()
if not isinstance(identifiers, list):
return []
for entry in identifiers:
if not isinstance(entry, dict):
continue
for kind, raw in entry.items():
if kind.lower().lstrip("/") not in {"gtin", "gtin8", "gtin12", "gtin13", "gtin14", "ean", "upc"}:
continue
for code in re.findall(r"(?<!\d)\d{8,14}(?!\d)", str(raw)):
if len(code) not in {8, 12, 13, 14} or len(set(code)) == 1:
continue
total = sum(int(digit) * (3 if i % 2 == 0 else 1) for i, digit in enumerate(reversed(code[:-1])))
if (10 - total % 10) % 10 == int(code[-1]):
result.add(code.zfill(14))
return sorted(result)
def clean_offer(record, seed=42, description_characters=500):
key = cluster_key(record.get("cluster_id"))
title = clean_text(record.get("title"))
if key is None or len(title) < 3:
return None
description_raw = record.get("description")
# Unwrapped JavaScript is not reliably separable from content: omit it.
contaminated = isinstance(description_raw, str) and any(marker in description_raw.lower() for marker in ("document.write", "javascript:"))
description = "" if contaminated else clean_text(description_raw)[:description_characters]
brand = clean_text(record.get("brand"))
text = title
if brand and brand.casefold() not in title.casefold():
text += " | " + brand
if description:
text += " | " + description
return {
"offer_id": str(record.get("id", "")),
"cluster_id": key,
"split": split_for_cluster(record["cluster_id"], seed),
"title": title,
"title_key": title.casefold(),
"text": text,
"brand": brand,
"category": clean_text(record.get("category")),
"gtins": canonical_gtins(record.get("identifiers")),
"language_tags": sorted(set(re.findall(r'"@([a-zA-Z]{2,3}(?:-[a-zA-Z0-9]+)*)\b', str(record.get("title", "")).lower()))),
}