MiniMax-H3-Image-mlx / scripts /publish_hf_release.py

Release v0.4.0: selective MLX model downloads

3da88e3 verified 2 days ago

13.2 kB

"""Publish verified release components after redistribution rights are established.
This command changes Hugging Face state. It never accepts model access requests
or sends announcements. Components may be uploaded one at a time to conserve
scratch space. The main repository is finalized only after every component.
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from h3_mlx.hub import read_json, sha256  # noqa: E402

PROJECT = "devin-lai/MiniMax-H3-Image-mlx"
RELEASE = "v0.4.0"
COMPONENTS = ("shared", "bf16", "packed12", "q8", "q4", "examples")


def write_json(path, value):
    path.write_text(json.dumps(value, indent=2) + "\n")


def verify_local(folder):
    manifest = read_json(folder / "files.json")
    if manifest.get("format") != "h3-mlx-files-v1":
        raise ValueError("Unknown file manifest")
    for name, spec in manifest["files"].items():
        from h3_mlx.hub import relative_path

        file = folder / relative_path(name)
        if not file.is_file() or file.stat().st_size != spec["bytes"]:
            raise ValueError(f"Missing or truncated local file: {file}")
        if sha256(file) != spec["sha256"]:
            raise ValueError(f"Local SHA-256 mismatch: {file}")
    quantization = folder / "model/core/quantization.json"
    if quantization.exists():
        files = read_json(quantization)["files"]
        if len(files) != 52:
            raise ValueError("Expected all fifty main blocks and two token refiners")
        for file in files:
            expected = manifest["files"].get("model/core/" + file["file"])
            if expected != {"bytes": file["bytes"], "sha256": file["sha256"]}:
                raise ValueError(f"Quantized export/manifest mismatch: {file['file']}")
    return manifest


def verify_remote(api, repo_id, manifest, *, repo_type="model"):
    from huggingface_hub import hf_hub_download

    info = api.repo_info(repo_id, repo_type=repo_type, files_metadata=True)
    files = {file.rfilename: file for file in info.siblings}
    for name, spec in manifest["files"].items():
        remote = files.get(name)
        if remote is None or remote.size != spec["bytes"]:
            raise ValueError(f"Remote file missing or wrong size: {repo_id}/{name}")
        lfs = getattr(remote, "lfs", None)
        digest = getattr(lfs, "sha256", None) if lfs is not None else None
        if isinstance(lfs, dict):
            digest = lfs.get("sha256")
        if digest is None:
            if remote.size > 10 * 1024**2:
                raise ValueError(
                    f"Cannot verify remote weight digest: {repo_id}/{name}"
                )
            digest = sha256(
                hf_hub_download(repo_id, name, revision=info.sha, repo_type=repo_type)
            )
        if digest != spec["sha256"]:
            raise ValueError(f"Remote SHA-256 mismatch: {repo_id}/{name}")
    return info.sha


def ensure_owned_repository(api, repo_id, folder, *, gated):
    from huggingface_hub import hf_hub_download

    marker = {"project": PROJECT, "release": RELEASE}
    if api.repo_exists(repo_id):
        try:
            existing = read_json(hf_hub_download(repo_id, "release-owner.json"))
        except Exception as error:
            raise RuntimeError(
                f"Existing repository has no matching release marker: {repo_id}"
            ) from error
        if existing != marker:
            raise RuntimeError(f"Refusing to modify unrelated repository: {repo_id}")
    else:
        api.create_repo(repo_id, private=True)
        # Establish ownership immediately so an interrupted upload is resumable.
        api.upload_file(
            repo_id=repo_id,
            path_in_repo="release-owner.json",
            path_or_fileobj=(json.dumps(marker) + "\n").encode(),
            commit_message=f"Initialize {RELEASE} release staging",
        )
    write_json(folder / "release-owner.json", marker)
    if gated:
        api.update_repo_settings(repo_id, gated="manual")


def refresh_small_file(folder, name):
    manifest = read_json(folder / "files.json")
    file = folder / name
    manifest["files"][name] = {"bytes": file.stat().st_size, "sha256": sha256(file)}
    write_json(folder / "files.json", manifest)


def publish_component(api, component, folder):
    repo_id = f"{PROJECT}-{component}"
    if component != "examples" and not (folder / "model").is_dir():
        raise ValueError(f"Model files are not prepared for {component}")
    if (
        component in ("q8", "q4")
        and not (folder / "model/core/quantization.json").is_file()
    ):
        raise ValueError(f"Quantized export incomplete: {component}")
    if (
        component == "packed12"
        and len(list((folder / "model/core").glob("blocks.*.safetensors"))) != 50
    ):
        raise ValueError("Expected all fifty packed blocks")
    card = (
        (folder / "README.md")
        .read_text()
        .replace(
            "**Preparation status:** this model repository is staged locally; publication\n"
            "and immutable commit IDs are pending confirmation of redistribution rights.",
            f"**Release:** {RELEASE}. Access requests are reviewed manually under the retained license.",
        )
        .replace(
            "**Preparation status:** staged locally; publication is pending confirmation\n"
            "of redistribution rights.",
            f"**Release:** {RELEASE}. Access requests are reviewed manually.",
        )
    )
    (folder / "README.md").write_text(card)
    refresh_small_file(folder, "README.md")
    manifest = verify_local(folder)
    ensure_owned_repository(api, repo_id, folder, gated=True)
    api.upload_large_folder(repo_id, folder, repo_type="model", num_workers=4)
    revision = verify_remote(api, repo_id, manifest)
    api.create_tag(repo_id, tag=RELEASE, revision=revision, exist_ok=True)
    if api.model_info(repo_id, revision=RELEASE).sha != revision:
        raise ValueError(f"Existing release tag points at another commit: {repo_id}")
    api.update_repo_settings(repo_id, gated="manual", private=False)
    info = api.model_info(repo_id)
    if info.private or info.gated != "manual":
        raise RuntimeError(f"Unexpected publication/access state: {repo_id}")
    return {
        "repo_id": repo_id,
        "revision": revision,
        "gated": "manual",
        "verified": True,
    }


def finalize_main(api, output, state):
    missing = [name for name in COMPONENTS if not state.get(name, {}).get("verified")]
    if missing:
        raise ValueError(
            f"Publish and verify these components first: {', '.join(missing)}"
        )
    from huggingface_hub import ModelCard

    catalog = read_json(ROOT / "h3_mlx/model_index.json")
    catalog["status"] = "published"
    catalog["shared"]["revision"] = state["shared"]["revision"]
    for name in catalog["variants"]:
        catalog["variants"][name]["revision"] = state[name]["revision"]
    main_folder = output / PROJECT.split("/")[-1]
    write_json(main_folder / "h3_mlx/model_index.json", catalog)
    collection = api.create_collection(
        "MiniMax H3 Image · Apple silicon / MLX",
        namespace="devin-lai",
        private=True,
        description="Local image generation and editing. BF16, lossless packed BF16, Q8 and Q4; selective downloads, shared components and measured comparisons.",
        exists_ok=True,
    )
    readme = (main_folder / "README.md").read_text()
    readme = readme.replace(
        "The current preparation has **not yet been\npublished** while redistribution rights are clarified. The commands below are\nthe prepared release workflow.",
        f"Release **{RELEASE}** is available with manual access review for model components.",
    )
    if f"[Collection]({collection.url})" not in readme:
        readme = readme.replace(
            "[Repository map](MODEL_INDEX.md)",
            f"[Collection]({collection.url}) · [Repository map](MODEL_INDEX.md)",
        )
    for name in ("q8", "q4"):
        files = read_json(output / f"MiniMax-H3-Image-mlx-{name}/files.json")["files"]
        size = (
            sum(
                spec["bytes"]
                for path, spec in files.items()
                if path.startswith("model/")
            )
            / 1024**3
        )
        readme = readme.replace(
            "About 20 GiB²" if name == "q8" else "About 11 GiB²", f"{size:.2f} GiB"
        )
    readme = readme.replace(
        " ² Estimates until the prepared export manifest is finalized.", ""
    )
    readme = readme.replace(
        " ² Q8 is an estimate until its export manifest is finalized.", ""
    )
    (main_folder / "README.md").write_text(readme)
    chinese = (
        (main_folder / "README.zh-CN.md")
        .read_text()
        .replace(
            "**发布状态：目前仅本地准备，尚未上传。**",
            f"**发布版本：{RELEASE}，模型组件采用人工审核访问。**",
        )
        .replace(
            "在确认发布方具有所需权限之前暂停公开发布。", "使用前请确认具备所需权限。"
        )
    )
    (main_folder / "README.zh-CN.md").write_text(chinese)
    release_file = main_folder / "RELEASE.md"
    release_notes = release_file.read_text()
    if "<!-- COLLECTION_LINK -->" in release_notes:
        release_notes = release_notes.replace(
            "<!-- COLLECTION_LINK -->", f"[Browse the collection]({collection.url})."
        )
    elif collection.url not in release_notes:
        release_notes += f"\n[Browse the collection]({collection.url}).\n"
    release_file.write_text(release_notes)
    ModelCard.load(main_folder / "README.md").validate()
    ensure_owned_repository(api, PROJECT, main_folder, gated=False)
    from scripts.prepare_hf_release import manifest as build_manifest

    build_manifest(main_folder, {})
    main_manifest = read_json(main_folder / "files.json")
    commit = api.upload_folder(
        repo_id=PROJECT,
        folder_path=main_folder,
        commit_message=f"Release {RELEASE}: selective MLX model downloads",
    )
    if verify_remote(api, PROJECT, main_manifest) != commit.oid:
        raise RuntimeError("Main repository changed during publication verification")
    api.create_tag(PROJECT, tag=RELEASE, revision=commit.oid, exist_ok=True)
    if api.model_info(PROJECT, revision=RELEASE).sha != commit.oid:
        raise ValueError("Main release tag points at another commit")
    api.update_repo_settings(PROJECT, private=False)
    notes = {
        "main": "Start here: runtime, quick start, model selection, authentic outputs and measured results.",
        "packed12": "Recommended exact storage choice: lossless BF16 packing, not quantization.",
        "bf16": "Unchanged original transformer files: native BF16/FP32 storage control.",
        "q8": "Smaller affine Q8 core; image changes and no established generation-speed advantage.",
        "q4": "Smallest affine Q4 core; stronger detail and lettering changes.",
        "shared": "VAE, native support, PDD heads, configurations and frozen six-step tables; reused across variants.",
        "examples": "Generated examples and visual comparisons, with access review under the retained license.",
    }
    for name in ("main", "packed12", "bf16", "q8", "q4", "shared", "examples"):
        repo_id = PROJECT if name == "main" else state[name]["repo_id"]
        api.add_collection_item(
            collection.slug, repo_id, "model", note=notes[name], exists_ok=True
        )
    api.update_collection_metadata(collection.slug, private=False, position=0)
    write_json(ROOT / "h3_mlx/model_index.json", catalog)
    return {
        "repo_id": PROJECT,
        "revision": commit.oid,
        "collection": collection.url,
        "verified": True,
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path, default=ROOT / ".release/huggingface")
    parser.add_argument("--component", choices=(*COMPONENTS, "main"), required=True)
    parser.add_argument(
        "--rights-confirmed",
        action="store_true",
        help="Publisher has established the required redistribution rights",
    )
    args = parser.parse_args()
    if not args.rights_confirmed:
        parser.error("Establish redistribution rights first; no Hub changes were made")
    from huggingface_hub import HfApi

    api = HfApi()
    if api.whoami()["name"] != "devin-lai":
        raise RuntimeError("Expected the devin-lai publishing account")
    state_path = args.output.resolve().parent / "publication.json"
    state = read_json(state_path) if state_path.exists() else {}
    if args.component == "main":
        state["main"] = finalize_main(api, args.output.resolve(), state)
    else:
        folder = args.output / f"MiniMax-H3-Image-mlx-{args.component}"
        state[args.component] = publish_component(api, args.component, folder)
    write_json(state_path, state)
    print(json.dumps(state[args.component], indent=2))


if __name__ == "__main__":
    main()

	"""Publish verified release components after redistribution rights are established.

	This command changes Hugging Face state. It never accepts model access requests
	or sends announcements. Components may be uploaded one at a time to conserve
	scratch space. The main repository is finalized only after every component.
	"""

	from __future__ import annotations

	import argparse
	import json
	import sys
	from pathlib import Path

	ROOT = Path(__file__).resolve().parents[1]
	sys.path.insert(0, str(ROOT))

	from h3_mlx.hub import read_json, sha256 # noqa: E402

	PROJECT = "devin-lai/MiniMax-H3-Image-mlx"
	RELEASE = "v0.4.0"
	COMPONENTS = ("shared", "bf16", "packed12", "q8", "q4", "examples")


	def write_json(path, value):
	path.write_text(json.dumps(value, indent=2) + "\n")


	def verify_local(folder):
	manifest = read_json(folder / "files.json")
	if manifest.get("format") != "h3-mlx-files-v1":
	raise ValueError("Unknown file manifest")
	for name, spec in manifest["files"].items():
	from h3_mlx.hub import relative_path

	file = folder / relative_path(name)
	if not file.is_file() or file.stat().st_size != spec["bytes"]:
	raise ValueError(f"Missing or truncated local file: {file}")
	if sha256(file) != spec["sha256"]:
	raise ValueError(f"Local SHA-256 mismatch: {file}")
	quantization = folder / "model/core/quantization.json"
	if quantization.exists():
	files = read_json(quantization)["files"]
	if len(files) != 52:
	raise ValueError("Expected all fifty main blocks and two token refiners")
	for file in files:
	expected = manifest["files"].get("model/core/" + file["file"])
	if expected != {"bytes": file["bytes"], "sha256": file["sha256"]}:
	raise ValueError(f"Quantized export/manifest mismatch: {file['file']}")
	return manifest


	def verify_remote(api, repo_id, manifest, *, repo_type="model"):
	from huggingface_hub import hf_hub_download

	info = api.repo_info(repo_id, repo_type=repo_type, files_metadata=True)
	files = {file.rfilename: file for file in info.siblings}
	for name, spec in manifest["files"].items():
	remote = files.get(name)
	if remote is None or remote.size != spec["bytes"]:
	raise ValueError(f"Remote file missing or wrong size: {repo_id}/{name}")
	lfs = getattr(remote, "lfs", None)
	digest = getattr(lfs, "sha256", None) if lfs is not None else None
	if isinstance(lfs, dict):
	digest = lfs.get("sha256")
	if digest is None:
	if remote.size > 10 * 1024**2:
	raise ValueError(
	f"Cannot verify remote weight digest: {repo_id}/{name}"
	)
	digest = sha256(
	hf_hub_download(repo_id, name, revision=info.sha, repo_type=repo_type)
	)
	if digest != spec["sha256"]:
	raise ValueError(f"Remote SHA-256 mismatch: {repo_id}/{name}")
	return info.sha


	def ensure_owned_repository(api, repo_id, folder, *, gated):
	from huggingface_hub import hf_hub_download

	marker = {"project": PROJECT, "release": RELEASE}
	if api.repo_exists(repo_id):
	try:
	existing = read_json(hf_hub_download(repo_id, "release-owner.json"))
	except Exception as error:
	raise RuntimeError(
	f"Existing repository has no matching release marker: {repo_id}"
	) from error
	if existing != marker:
	raise RuntimeError(f"Refusing to modify unrelated repository: {repo_id}")
	else:
	api.create_repo(repo_id, private=True)
	# Establish ownership immediately so an interrupted upload is resumable.
	api.upload_file(
	repo_id=repo_id,
	path_in_repo="release-owner.json",
	path_or_fileobj=(json.dumps(marker) + "\n").encode(),
	commit_message=f"Initialize {RELEASE} release staging",
	)
	write_json(folder / "release-owner.json", marker)
	if gated:
	api.update_repo_settings(repo_id, gated="manual")


	def refresh_small_file(folder, name):
	manifest = read_json(folder / "files.json")
	file = folder / name
	manifest["files"][name] = {"bytes": file.stat().st_size, "sha256": sha256(file)}
	write_json(folder / "files.json", manifest)


	def publish_component(api, component, folder):
	repo_id = f"{PROJECT}-{component}"
	if component != "examples" and not (folder / "model").is_dir():
	raise ValueError(f"Model files are not prepared for {component}")
	if (
	component in ("q8", "q4")
	and not (folder / "model/core/quantization.json").is_file()
	):
	raise ValueError(f"Quantized export incomplete: {component}")
	if (
	component == "packed12"
	and len(list((folder / "model/core").glob("blocks.*.safetensors"))) != 50
	):
	raise ValueError("Expected all fifty packed blocks")
	card = (
	(folder / "README.md")
	.read_text()
	.replace(
	"Preparation status: this model repository is staged locally; publication\n"
	"and immutable commit IDs are pending confirmation of redistribution rights.",
	f"Release: {RELEASE}. Access requests are reviewed manually under the retained license.",
	)
	.replace(
	"Preparation status: staged locally; publication is pending confirmation\n"
	"of redistribution rights.",
	f"Release: {RELEASE}. Access requests are reviewed manually.",
	)
	)
	(folder / "README.md").write_text(card)
	refresh_small_file(folder, "README.md")
	manifest = verify_local(folder)
	ensure_owned_repository(api, repo_id, folder, gated=True)
	api.upload_large_folder(repo_id, folder, repo_type="model", num_workers=4)
	revision = verify_remote(api, repo_id, manifest)
	api.create_tag(repo_id, tag=RELEASE, revision=revision, exist_ok=True)
	if api.model_info(repo_id, revision=RELEASE).sha != revision:
	raise ValueError(f"Existing release tag points at another commit: {repo_id}")
	api.update_repo_settings(repo_id, gated="manual", private=False)
	info = api.model_info(repo_id)
	if info.private or info.gated != "manual":
	raise RuntimeError(f"Unexpected publication/access state: {repo_id}")
	return {
	"repo_id": repo_id,
	"revision": revision,
	"gated": "manual",
	"verified": True,
	}


	def finalize_main(api, output, state):
	missing = [name for name in COMPONENTS if not state.get(name, {}).get("verified")]
	if missing:
	raise ValueError(
	f"Publish and verify these components first: {', '.join(missing)}"
	)
	from huggingface_hub import ModelCard

	catalog = read_json(ROOT / "h3_mlx/model_index.json")
	catalog["status"] = "published"
	catalog["shared"]["revision"] = state["shared"]["revision"]
	for name in catalog["variants"]:
	catalog["variants"][name]["revision"] = state[name]["revision"]
	main_folder = output / PROJECT.split("/")[-1]
	write_json(main_folder / "h3_mlx/model_index.json", catalog)
	collection = api.create_collection(
	"MiniMax H3 Image · Apple silicon / MLX",
	namespace="devin-lai",
	private=True,
	description="Local image generation and editing. BF16, lossless packed BF16, Q8 and Q4; selective downloads, shared components and measured comparisons.",
	exists_ok=True,
	)
	readme = (main_folder / "README.md").read_text()
	readme = readme.replace(
	"The current preparation has not yet been\npublished while redistribution rights are clarified. The commands below are\nthe prepared release workflow.",
	f"Release {RELEASE} is available with manual access review for model components.",
	)
	if f"[Collection]({collection.url})" not in readme:
	readme = readme.replace(
	"[Repository map](MODEL_INDEX.md)",
	f"[Collection]({collection.url}) · [Repository map](MODEL_INDEX.md)",
	)
	for name in ("q8", "q4"):
	files = read_json(output / f"MiniMax-H3-Image-mlx-{name}/files.json")["files"]
	size = (
	sum(
	spec["bytes"]
	for path, spec in files.items()
	if path.startswith("model/")
	)
	/ 1024**3
	)
	readme = readme.replace(
	"About 20 GiB²" if name == "q8" else "About 11 GiB²", f"{size:.2f} GiB"
	)
	readme = readme.replace(
	" ² Estimates until the prepared export manifest is finalized.", ""
	)
	readme = readme.replace(
	" ² Q8 is an estimate until its export manifest is finalized.", ""
	)
	(main_folder / "README.md").write_text(readme)
	chinese = (
	(main_folder / "README.zh-CN.md")
	.read_text()
	.replace(
	"发布状态：目前仅本地准备，尚未上传。",
	f"发布版本：{RELEASE}，模型组件采用人工审核访问。",
	)
	.replace(
	"在确认发布方具有所需权限之前暂停公开发布。", "使用前请确认具备所需权限。"
	)
	)
	(main_folder / "README.zh-CN.md").write_text(chinese)
	release_file = main_folder / "RELEASE.md"
	release_notes = release_file.read_text()
	if "<!-- COLLECTION_LINK -->" in release_notes:
	release_notes = release_notes.replace(
	"<!-- COLLECTION_LINK -->", f"[Browse the collection]({collection.url})."
	)
	elif collection.url not in release_notes:
	release_notes += f"\n[Browse the collection]({collection.url}).\n"
	release_file.write_text(release_notes)
	ModelCard.load(main_folder / "README.md").validate()
	ensure_owned_repository(api, PROJECT, main_folder, gated=False)
	from scripts.prepare_hf_release import manifest as build_manifest

	build_manifest(main_folder, {})
	main_manifest = read_json(main_folder / "files.json")
	commit = api.upload_folder(
	repo_id=PROJECT,
	folder_path=main_folder,
	commit_message=f"Release {RELEASE}: selective MLX model downloads",
	)
	if verify_remote(api, PROJECT, main_manifest) != commit.oid:
	raise RuntimeError("Main repository changed during publication verification")
	api.create_tag(PROJECT, tag=RELEASE, revision=commit.oid, exist_ok=True)
	if api.model_info(PROJECT, revision=RELEASE).sha != commit.oid:
	raise ValueError("Main release tag points at another commit")
	api.update_repo_settings(PROJECT, private=False)
	notes = {
	"main": "Start here: runtime, quick start, model selection, authentic outputs and measured results.",
	"packed12": "Recommended exact storage choice: lossless BF16 packing, not quantization.",
	"bf16": "Unchanged original transformer files: native BF16/FP32 storage control.",
	"q8": "Smaller affine Q8 core; image changes and no established generation-speed advantage.",
	"q4": "Smallest affine Q4 core; stronger detail and lettering changes.",
	"shared": "VAE, native support, PDD heads, configurations and frozen six-step tables; reused across variants.",
	"examples": "Generated examples and visual comparisons, with access review under the retained license.",
	}
	for name in ("main", "packed12", "bf16", "q8", "q4", "shared", "examples"):
	repo_id = PROJECT if name == "main" else state[name]["repo_id"]
	api.add_collection_item(
	collection.slug, repo_id, "model", note=notes[name], exists_ok=True
	)
	api.update_collection_metadata(collection.slug, private=False, position=0)
	write_json(ROOT / "h3_mlx/model_index.json", catalog)
	return {
	"repo_id": PROJECT,
	"revision": commit.oid,
	"collection": collection.url,
	"verified": True,
	}


	def main():
	parser = argparse.ArgumentParser(description=__doc__)
	parser.add_argument("--output", type=Path, default=ROOT / ".release/huggingface")
	parser.add_argument("--component", choices=(*COMPONENTS, "main"), required=True)
	parser.add_argument(
	"--rights-confirmed",
	action="store_true",
	help="Publisher has established the required redistribution rights",
	)
	args = parser.parse_args()
	if not args.rights_confirmed:
	parser.error("Establish redistribution rights first; no Hub changes were made")
	from huggingface_hub import HfApi

	api = HfApi()
	if api.whoami()["name"] != "devin-lai":
	raise RuntimeError("Expected the devin-lai publishing account")
	state_path = args.output.resolve().parent / "publication.json"
	state = read_json(state_path) if state_path.exists() else {}
	if args.component == "main":
	state["main"] = finalize_main(api, args.output.resolve(), state)
	else:
	folder = args.output / f"MiniMax-H3-Image-mlx-{args.component}"
	state[args.component] = publish_component(api, args.component, folder)
	write_json(state_path, state)
	print(json.dumps(state[args.component], indent=2))


	if __name__ == "__main__":
	main()