Sentence Similarity
sentence-transformers
Safetensors
xlm-roberta
product-similarity
product-embeddings
weak-supervision
text-embeddings-inference
Instructions to use beno-rohman/product-embedding-e5-base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- sentence-transformers
How to use beno-rohman/product-embedding-e5-base with sentence-transformers:
from sentence_transformers import SentenceTransformer model = SentenceTransformer("beno-rohman/product-embedding-e5-base") sentences = [ "That is a happy person", "That is a happy dog", "That is a very happy person", "Today is a sunny day" ] embeddings = model.encode(sentences) similarities = model.similarity(embeddings, embeddings) print(similarities.shape) # [4, 4] - Notebooks
- Google Colab
- Kaggle
Download training/scripts/publish_model.py from beno-rohman/product-embedding-e5-base: direct link, hf CLI and curl.
- Browser
- Download file 5.94 kB
-
https://huggingface.co/beno-rohman/product-embedding-e5-base/resolve/main/training/scripts/publish_model.py
- Command line
-
hf download hf://beno-rohman/product-embedding-e5-base/training/scripts/publish_model.py
-
curl -L -o publish_model.py https://huggingface.co/beno-rohman/product-embedding-e5-base/resolve/main/training/scripts/publish_model.py
5.94 kB
| """Publish an explicitly built model release. Token is read only from HF_TOKEN.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| from huggingface_hub import HfApi | |
| def publish(folder, *, execute=False, update=False, private=None, expected_revision=None): | |
| manifest = json.loads((folder / "release-manifest.json").read_text(encoding="utf-8")) | |
| repo_id = manifest["repo_id"] | |
| expected = set(manifest["files"]) | {"release-manifest.json"} | |
| actual = {path.relative_to(folder).as_posix() for path in folder.rglob("*") if path.is_file()} | |
| if actual != expected: | |
| raise ValueError("Release file list changed; rebuild before uploading") | |
| for name, information in manifest["files"].items(): | |
| digest = hashlib.sha256() | |
| with (folder / name).open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| if digest.hexdigest() != information["sha256"]: | |
| raise ValueError(f"Checksum mismatch: {name}") | |
| if not execute: | |
| requested_private = private if private is not None else manifest.get("private") | |
| visibility = "preserve existing" if requested_private is None else "private" if requested_private else "public" | |
| print(json.dumps({"status": "dry_run", "repo_id": repo_id, "visibility": visibility, | |
| "license": manifest["license"], "verified_files": len(actual), | |
| "superseded_paths": manifest.get("superseded_paths", [])}, indent=2)) | |
| return | |
| token = os.environ.get("HF_TOKEN") | |
| if not token: | |
| raise ValueError("Set HF_TOKEN in the process environment before publishing") | |
| api = HfApi(token=token) | |
| account = api.whoami()["name"] | |
| if repo_id.split("/")[0] != account: | |
| raise ValueError("Release namespace must match the authenticated personal account") | |
| exists = api.repo_exists(repo_id, repo_type="model") | |
| if exists and not update: | |
| raise FileExistsError("Repository already exists; review it before explicitly using --update") | |
| requested_private = private if private is not None else manifest.get("private") | |
| previous = api.model_info(repo_id) if exists else None | |
| if previous is not None: | |
| if expected_revision is not None and previous.sha != expected_revision: | |
| raise ValueError("Repository revision changed; inspect the remote changes before updating") | |
| expected_private = previous.private | |
| if requested_private is not None and requested_private != expected_private: | |
| raise ValueError("Requested visibility does not match the existing repository; visibility is preserved") | |
| else: | |
| expected_private = bool(requested_private) | |
| api.create_repo(repo_id, repo_type="model", private=expected_private) | |
| existing_paths = {item.rfilename for item in previous.siblings} if previous is not None else set() | |
| superseded = sorted(set(manifest.get("superseded_paths", [])) & existing_paths) | |
| commit_message = ("Update product-function similarity checkpoint and concise model card" | |
| if manifest.get("variant") == "product-type" else "Publish WDC product embedding checkpoint, training source, and evaluation") | |
| commit = api.upload_folder(repo_id=repo_id, repo_type="model", folder_path=folder, | |
| commit_message=commit_message, parent_commit=previous.sha if previous else None, | |
| delete_patterns=superseded or None) | |
| info = api.model_info(repo_id, files_metadata=True) | |
| remote = {item.rfilename: item for item in info.siblings} | |
| if not expected.issubset(remote): | |
| raise RuntimeError("Some release files are absent from the published repository") | |
| weights = remote.get("model.safetensors") | |
| if info.private != expected_private or info.card_data.get("license") != "mit" or weights is None: | |
| raise RuntimeError("Published repository metadata verification failed") | |
| lfs = weights.lfs | |
| remote_sha = getattr(lfs, "sha256", None) if lfs is not None else None | |
| if remote_sha and remote_sha != manifest["files"]["model.safetensors"]["sha256"]: | |
| raise RuntimeError("Remote model weight checksum mismatch") | |
| if weights.size != manifest["files"]["model.safetensors"]["bytes"]: | |
| raise RuntimeError("Remote model weight size mismatch") | |
| receipt = {"repo_id": repo_id, "url": f"https://huggingface.co/{repo_id}", | |
| "revision": info.sha, "commit_url": str(commit.commit_url), "public": not info.private, | |
| "private": info.private, "previous_revision": previous.sha if previous else None, | |
| "variant": manifest.get("variant", "identity"), "superseded_paths_removed": superseded, | |
| "license": info.card_data.get("license"), "remote_files": len(remote), | |
| "weights_sha256": remote_sha, "token_saved": False} | |
| (folder.parent / "huggingface-upload.json").write_text(json.dumps(receipt, indent=2) + "\n") | |
| print(json.dumps(receipt, indent=2)) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--folder", type=Path, default=Path("artifacts/hf-release")) | |
| parser.add_argument("--execute", action="store_true") | |
| parser.add_argument("--update", action="store_true") | |
| parser.add_argument("--private", action="store_const", const=True, default=None) | |
| parser.add_argument("--expected-revision") | |
| args = parser.parse_args() | |
| try: | |
| publish(args.folder, execute=args.execute, update=args.update, private=args.private, | |
| expected_revision=args.expected_revision) | |
| except Exception as exc: | |
| # Avoid echoing credentials if an upstream exception ever includes them. | |
| message = re.sub(r"hf_[A-Za-z0-9]{20,}", "[REDACTED]", str(exc)) | |
| parser.exit(1, f"{type(exc).__name__}: {message}\n") | |