Sentence Similarity
sentence-transformers
Safetensors
xlm-roberta
product-similarity
product-embeddings
weak-supervision
text-embeddings-inference
Instructions to use beno-rohman/product-embedding-e5-base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- sentence-transformers
How to use beno-rohman/product-embedding-e5-base with sentence-transformers:
from sentence_transformers import SentenceTransformer model = SentenceTransformer("beno-rohman/product-embedding-e5-base") sentences = [ "That is a happy person", "That is a happy dog", "That is a very happy person", "Today is a sunny day" ] embeddings = model.encode(sentences) similarities = model.similarity(embeddings, embeddings) print(similarities.shape) # [4, 4] - Notebooks
- Google Colab
- Kaggle
Download training/src/product_embedding/data.py from beno-rohman/product-embedding-e5-base: direct link, hf CLI and curl.
- Browser
- Download file 3.37 kB
-
https://huggingface.co/beno-rohman/product-embedding-e5-base/resolve/main/training/src/product_embedding/data.py
- Command line
-
hf download hf://beno-rohman/product-embedding-e5-base/training/src/product_embedding/data.py
-
curl -L -o data.py https://huggingface.co/beno-rohman/product-embedding-e5-base/resolve/main/training/src/product_embedding/data.py
3.37 kB
| """Conservative text cleaning and canonical identifiers.""" | |
| from html.parser import HTMLParser | |
| import re | |
| from .profile import cluster_key, normalize_text, split_for_cluster | |
| class VisibleText(HTMLParser): | |
| def __init__(self): | |
| super().__init__(convert_charrefs=True) | |
| self.parts = [] | |
| self.hidden = [] | |
| def handle_starttag(self, tag, attrs): | |
| if tag in {"script", "style", "noscript"}: | |
| self.hidden.append(tag) | |
| elif not self.hidden: | |
| self.parts.append(" ") | |
| def handle_endtag(self, tag): | |
| if self.hidden: | |
| if tag == self.hidden[-1]: | |
| self.hidden.pop() | |
| else: | |
| self.parts.append(" ") | |
| def handle_data(self, data): | |
| if not self.hidden: | |
| self.parts.append(data) | |
| def clean_text(value): | |
| if not isinstance(value, str): | |
| return "" | |
| # Do not parse ordinary model-number comparisons as HTML. | |
| if re.search(r"</?[a-zA-Z][^>]*>", value): | |
| parser = VisibleText() | |
| parser.feed(value) | |
| value = "".join(parser.parts) | |
| value = re.sub(r'(?<=")@[a-zA-Z]{2,3}(?:-[a-zA-Z0-9]+)*\b', "", value) | |
| return normalize_text(value) | |
| def canonical_gtins(identifiers): | |
| """Only valid GTIN checksums; seller-local SKUs are never global IDs.""" | |
| result = set() | |
| if not isinstance(identifiers, list): | |
| return [] | |
| for entry in identifiers: | |
| if not isinstance(entry, dict): | |
| continue | |
| for kind, raw in entry.items(): | |
| if kind.lower().lstrip("/") not in {"gtin", "gtin8", "gtin12", "gtin13", "gtin14", "ean", "upc"}: | |
| continue | |
| for code in re.findall(r"(?<!\d)\d{8,14}(?!\d)", str(raw)): | |
| if len(code) not in {8, 12, 13, 14} or len(set(code)) == 1: | |
| continue | |
| total = sum(int(digit) * (3 if i % 2 == 0 else 1) for i, digit in enumerate(reversed(code[:-1]))) | |
| if (10 - total % 10) % 10 == int(code[-1]): | |
| result.add(code.zfill(14)) | |
| return sorted(result) | |
| def clean_offer(record, seed=42, description_characters=500): | |
| key = cluster_key(record.get("cluster_id")) | |
| title = clean_text(record.get("title")) | |
| if key is None or len(title) < 3: | |
| return None | |
| description_raw = record.get("description") | |
| # Unwrapped JavaScript is not reliably separable from content: omit it. | |
| contaminated = isinstance(description_raw, str) and any(marker in description_raw.lower() for marker in ("document.write", "javascript:")) | |
| description = "" if contaminated else clean_text(description_raw)[:description_characters] | |
| brand = clean_text(record.get("brand")) | |
| text = title | |
| if brand and brand.casefold() not in title.casefold(): | |
| text += " | " + brand | |
| if description: | |
| text += " | " + description | |
| return { | |
| "offer_id": str(record.get("id", "")), | |
| "cluster_id": key, | |
| "split": split_for_cluster(record["cluster_id"], seed), | |
| "title": title, | |
| "title_key": title.casefold(), | |
| "text": text, | |
| "brand": brand, | |
| "category": clean_text(record.get("category")), | |
| "gtins": canonical_gtins(record.get("identifiers")), | |
| "language_tags": sorted(set(re.findall(r'"@([a-zA-Z]{2,3}(?:-[a-zA-Z0-9]+)*)\b', str(record.get("title", "")).lower()))), | |
| } | |