Inkling-2-Mini-MLX-2bit / extras /scripts /build_variant.py
VisualInference's picture
Inkling 2 Mini for MLX, 2bit: weights, card, validation results
031489f verified
Raw History Blame Contribute Delete
3.73 kB
"""Build a publishable mlx-vlm checkpoint directory from a recipe.
Usage: build_variant.py <recipes.json> <recipe_name> <out_dir> [--qdir q/shards]
Writes weights (mlx-vlm layout), config.json with explicit per-module quantization, tokenizer and processor files,
the bundled DSpark drafter (mtp.safetensors, unchanged) and a build manifest; then reloads the directory and checks
that its logits equal the in-memory model's.
"""
import hashlib
import json
import os
import shutil
import sys
import mlx.core as mx
import numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab # noqa: E402
import recipes as R # noqa: E402
from mlx_vlm import load # noqa: E402
from mlx_vlm.utils import save_config, save_weights # noqa: E402
COPY = ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja",
"processor_config.json", "generation_config.json", "mtp.safetensors")
UPSTREAM_REVISION = "ed40e7da625f594fc9c4990f5f66be24873c22d7"
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(16 << 20):
h.update(chunk)
return h.hexdigest()
def main():
args = sys.argv[1:]
qdir = os.path.join(qlab.ROOT, "q", "shards")
if "--qdir" in args:
i = args.index("--qdir"); qdir = args[i + 1]; del args[i:i + 2]
recipe_path, name, out_dir = args[:3]
recipe = json.load(open(recipe_path))[name]
os.makedirs(out_dir, exist_ok=True)
model, processor = qlab.load_native()
probe = json.load(open(os.path.join(qlab.ROOT, "parity/ref_fp32/tool_roundtrip.ids.json")))["ids"]
overrides = R.apply(model, recipe, qdir, log=print)
before = np.array(qlab.logits_for(model, probe))
config = json.load(open(os.path.join(qlab.NATIVE, "config.json")))
quant = {"group_size": 64, "bits": 8, "mode": "affine"}
quant.update({k: overrides[k] for k in sorted(overrides)})
config["quantization"] = quant
config["quantization_config"] = quant
save_weights(out_dir, model)
save_config(config, os.path.join(out_dir, "config.json"))
for f in COPY:
shutil.copy2(os.path.join(qlab.NATIVE, f), os.path.join(out_dir, f))
os.makedirs(os.path.join(out_dir, "tiktoken"), exist_ok=True)
shutil.copy2(os.path.join(qlab.NATIVE, "tiktoken", "tokenizer.model"), os.path.join(out_dir, "tiktoken", "tokenizer.model"))
kinds = {}
for path, q in overrides.items():
group = "experts" if ".switch_mlp." in path else R.dense_group(path, None) or "other"
key = f"{group}: {q['mode']} {q['bits']}-bit g{q['group_size']}"
kinds[key] = kinds.get(key, 0) + 1
del model
mx.clear_cache()
reloaded, _ = load(out_dir)
after = np.array(qlab.logits_for(reloaded, probe))
max_diff = float(np.abs(before - after).max())
files = sorted(f for f in os.listdir(out_dir) if f.endswith(".safetensors"))
manifest = dict(
name=name, recipe=recipe, upstream="thinkingmachines/Inkling-2-Mini", upstream_revision=UPSTREAM_REVISION,
module_counts=kinds, reload_max_logit_diff=max_diff,
weights_bytes=sum(os.path.getsize(os.path.join(out_dir, f)) for f in files if f != "mtp.safetensors"),
files={f: dict(bytes=os.path.getsize(os.path.join(out_dir, f)), sha256=sha256(os.path.join(out_dir, f))) for f in files},
)
json.dump(manifest, open(os.path.join(out_dir, "build_manifest.json"), "w"), indent=1)
print(json.dumps({k: v for k, v in manifest.items() if k not in ("files", "recipe")}, indent=1))
if max_diff != 0.0:
raise SystemExit(f"reloaded checkpoint differs from the in-memory model (max logit diff {max_diff})")
if __name__ == "__main__":
main()