beno-rohman's picture
Update product-function similarity checkpoint and concise model card
4aafc75 verified
Raw History Blame Contribute Delete
5.94 kB
"""Publish an explicitly built model release. Token is read only from HF_TOKEN."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
from huggingface_hub import HfApi
def publish(folder, *, execute=False, update=False, private=None, expected_revision=None):
manifest = json.loads((folder / "release-manifest.json").read_text(encoding="utf-8"))
repo_id = manifest["repo_id"]
expected = set(manifest["files"]) | {"release-manifest.json"}
actual = {path.relative_to(folder).as_posix() for path in folder.rglob("*") if path.is_file()}
if actual != expected:
raise ValueError("Release file list changed; rebuild before uploading")
for name, information in manifest["files"].items():
digest = hashlib.sha256()
with (folder / name).open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
if digest.hexdigest() != information["sha256"]:
raise ValueError(f"Checksum mismatch: {name}")
if not execute:
requested_private = private if private is not None else manifest.get("private")
visibility = "preserve existing" if requested_private is None else "private" if requested_private else "public"
print(json.dumps({"status": "dry_run", "repo_id": repo_id, "visibility": visibility,
"license": manifest["license"], "verified_files": len(actual),
"superseded_paths": manifest.get("superseded_paths", [])}, indent=2))
return
token = os.environ.get("HF_TOKEN")
if not token:
raise ValueError("Set HF_TOKEN in the process environment before publishing")
api = HfApi(token=token)
account = api.whoami()["name"]
if repo_id.split("/")[0] != account:
raise ValueError("Release namespace must match the authenticated personal account")
exists = api.repo_exists(repo_id, repo_type="model")
if exists and not update:
raise FileExistsError("Repository already exists; review it before explicitly using --update")
requested_private = private if private is not None else manifest.get("private")
previous = api.model_info(repo_id) if exists else None
if previous is not None:
if expected_revision is not None and previous.sha != expected_revision:
raise ValueError("Repository revision changed; inspect the remote changes before updating")
expected_private = previous.private
if requested_private is not None and requested_private != expected_private:
raise ValueError("Requested visibility does not match the existing repository; visibility is preserved")
else:
expected_private = bool(requested_private)
api.create_repo(repo_id, repo_type="model", private=expected_private)
existing_paths = {item.rfilename for item in previous.siblings} if previous is not None else set()
superseded = sorted(set(manifest.get("superseded_paths", [])) & existing_paths)
commit_message = ("Update product-function similarity checkpoint and concise model card"
if manifest.get("variant") == "product-type" else "Publish WDC product embedding checkpoint, training source, and evaluation")
commit = api.upload_folder(repo_id=repo_id, repo_type="model", folder_path=folder,
commit_message=commit_message, parent_commit=previous.sha if previous else None,
delete_patterns=superseded or None)
info = api.model_info(repo_id, files_metadata=True)
remote = {item.rfilename: item for item in info.siblings}
if not expected.issubset(remote):
raise RuntimeError("Some release files are absent from the published repository")
weights = remote.get("model.safetensors")
if info.private != expected_private or info.card_data.get("license") != "mit" or weights is None:
raise RuntimeError("Published repository metadata verification failed")
lfs = weights.lfs
remote_sha = getattr(lfs, "sha256", None) if lfs is not None else None
if remote_sha and remote_sha != manifest["files"]["model.safetensors"]["sha256"]:
raise RuntimeError("Remote model weight checksum mismatch")
if weights.size != manifest["files"]["model.safetensors"]["bytes"]:
raise RuntimeError("Remote model weight size mismatch")
receipt = {"repo_id": repo_id, "url": f"https://huggingface.co/{repo_id}",
"revision": info.sha, "commit_url": str(commit.commit_url), "public": not info.private,
"private": info.private, "previous_revision": previous.sha if previous else None,
"variant": manifest.get("variant", "identity"), "superseded_paths_removed": superseded,
"license": info.card_data.get("license"), "remote_files": len(remote),
"weights_sha256": remote_sha, "token_saved": False}
(folder.parent / "huggingface-upload.json").write_text(json.dumps(receipt, indent=2) + "\n")
print(json.dumps(receipt, indent=2))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--folder", type=Path, default=Path("artifacts/hf-release"))
parser.add_argument("--execute", action="store_true")
parser.add_argument("--update", action="store_true")
parser.add_argument("--private", action="store_const", const=True, default=None)
parser.add_argument("--expected-revision")
args = parser.parse_args()
try:
publish(args.folder, execute=args.execute, update=args.update, private=args.private,
expected_revision=args.expected_revision)
except Exception as exc:
# Avoid echoing credentials if an upstream exception ever includes them.
message = re.sub(r"hf_[A-Za-z0-9]{20,}", "[REDACTED]", str(exc))
parser.exit(1, f"{type(exc).__name__}: {message}\n")