"""Publish an explicitly built model release. Token is read only from HF_TOKEN.""" import argparse import hashlib import json import os from pathlib import Path import re from huggingface_hub import HfApi def publish(folder, *, execute=False, update=False, private=None, expected_revision=None): manifest = json.loads((folder / "release-manifest.json").read_text(encoding="utf-8")) repo_id = manifest["repo_id"] expected = set(manifest["files"]) | {"release-manifest.json"} actual = {path.relative_to(folder).as_posix() for path in folder.rglob("*") if path.is_file()} if actual != expected: raise ValueError("Release file list changed; rebuild before uploading") for name, information in manifest["files"].items(): digest = hashlib.sha256() with (folder / name).open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) if digest.hexdigest() != information["sha256"]: raise ValueError(f"Checksum mismatch: {name}") if not execute: requested_private = private if private is not None else manifest.get("private") visibility = "preserve existing" if requested_private is None else "private" if requested_private else "public" print(json.dumps({"status": "dry_run", "repo_id": repo_id, "visibility": visibility, "license": manifest["license"], "verified_files": len(actual), "superseded_paths": manifest.get("superseded_paths", [])}, indent=2)) return token = os.environ.get("HF_TOKEN") if not token: raise ValueError("Set HF_TOKEN in the process environment before publishing") api = HfApi(token=token) account = api.whoami()["name"] if repo_id.split("/")[0] != account: raise ValueError("Release namespace must match the authenticated personal account") exists = api.repo_exists(repo_id, repo_type="model") if exists and not update: raise FileExistsError("Repository already exists; review it before explicitly using --update") requested_private = private if private is not None else manifest.get("private") previous = api.model_info(repo_id) if exists else None if previous is not None: if expected_revision is not None and previous.sha != expected_revision: raise ValueError("Repository revision changed; inspect the remote changes before updating") expected_private = previous.private if requested_private is not None and requested_private != expected_private: raise ValueError("Requested visibility does not match the existing repository; visibility is preserved") else: expected_private = bool(requested_private) api.create_repo(repo_id, repo_type="model", private=expected_private) existing_paths = {item.rfilename for item in previous.siblings} if previous is not None else set() superseded = sorted(set(manifest.get("superseded_paths", [])) & existing_paths) commit_message = ("Update product-function similarity checkpoint and concise model card" if manifest.get("variant") == "product-type" else "Publish WDC product embedding checkpoint, training source, and evaluation") commit = api.upload_folder(repo_id=repo_id, repo_type="model", folder_path=folder, commit_message=commit_message, parent_commit=previous.sha if previous else None, delete_patterns=superseded or None) info = api.model_info(repo_id, files_metadata=True) remote = {item.rfilename: item for item in info.siblings} if not expected.issubset(remote): raise RuntimeError("Some release files are absent from the published repository") weights = remote.get("model.safetensors") if info.private != expected_private or info.card_data.get("license") != "mit" or weights is None: raise RuntimeError("Published repository metadata verification failed") lfs = weights.lfs remote_sha = getattr(lfs, "sha256", None) if lfs is not None else None if remote_sha and remote_sha != manifest["files"]["model.safetensors"]["sha256"]: raise RuntimeError("Remote model weight checksum mismatch") if weights.size != manifest["files"]["model.safetensors"]["bytes"]: raise RuntimeError("Remote model weight size mismatch") receipt = {"repo_id": repo_id, "url": f"https://huggingface.co/{repo_id}", "revision": info.sha, "commit_url": str(commit.commit_url), "public": not info.private, "private": info.private, "previous_revision": previous.sha if previous else None, "variant": manifest.get("variant", "identity"), "superseded_paths_removed": superseded, "license": info.card_data.get("license"), "remote_files": len(remote), "weights_sha256": remote_sha, "token_saved": False} (folder.parent / "huggingface-upload.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--folder", type=Path, default=Path("artifacts/hf-release")) parser.add_argument("--execute", action="store_true") parser.add_argument("--update", action="store_true") parser.add_argument("--private", action="store_const", const=True, default=None) parser.add_argument("--expected-revision") args = parser.parse_args() try: publish(args.folder, execute=args.execute, update=args.update, private=args.private, expected_revision=args.expected_revision) except Exception as exc: # Avoid echoing credentials if an upstream exception ever includes them. message = re.sub(r"hf_[A-Za-z0-9]{20,}", "[REDACTED]", str(exc)) parser.exit(1, f"{type(exc).__name__}: {message}\n")