File size: 3,733 Bytes
d19188c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Build a publishable mlx-vlm checkpoint directory from a recipe.

Usage: build_variant.py <recipes.json> <recipe_name> <out_dir> [--qdir q/shards]
Writes weights (mlx-vlm layout), config.json with explicit per-module quantization, tokenizer and processor files,
the bundled DSpark drafter (mtp.safetensors, unchanged) and a build manifest; then reloads the directory and checks
that its logits equal the in-memory model's.
"""

import hashlib
import json
import os
import shutil
import sys

import mlx.core as mx
import numpy as np

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab  # noqa: E402
import recipes as R  # noqa: E402
from mlx_vlm import load  # noqa: E402
from mlx_vlm.utils import save_config, save_weights  # noqa: E402

COPY = ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja",
        "processor_config.json", "generation_config.json", "mtp.safetensors")
UPSTREAM_REVISION = "ed40e7da625f594fc9c4990f5f66be24873c22d7"


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        while chunk := f.read(16 << 20):
            h.update(chunk)
    return h.hexdigest()


def main():
    args = sys.argv[1:]
    qdir = os.path.join(qlab.ROOT, "q", "shards")
    if "--qdir" in args:
        i = args.index("--qdir"); qdir = args[i + 1]; del args[i:i + 2]
    recipe_path, name, out_dir = args[:3]
    recipe = json.load(open(recipe_path))[name]
    os.makedirs(out_dir, exist_ok=True)
    model, processor = qlab.load_native()
    probe = json.load(open(os.path.join(qlab.ROOT, "parity/ref_fp32/tool_roundtrip.ids.json")))["ids"]
    overrides = R.apply(model, recipe, qdir, log=print)
    before = np.array(qlab.logits_for(model, probe))

    config = json.load(open(os.path.join(qlab.NATIVE, "config.json")))
    quant = {"group_size": 64, "bits": 8, "mode": "affine"}
    quant.update({k: overrides[k] for k in sorted(overrides)})
    config["quantization"] = quant
    config["quantization_config"] = quant
    save_weights(out_dir, model)
    save_config(config, os.path.join(out_dir, "config.json"))
    for f in COPY:
        shutil.copy2(os.path.join(qlab.NATIVE, f), os.path.join(out_dir, f))
    os.makedirs(os.path.join(out_dir, "tiktoken"), exist_ok=True)
    shutil.copy2(os.path.join(qlab.NATIVE, "tiktoken", "tokenizer.model"), os.path.join(out_dir, "tiktoken", "tokenizer.model"))

    kinds = {}
    for path, q in overrides.items():
        group = "experts" if ".switch_mlp." in path else R.dense_group(path, None) or "other"
        key = f"{group}: {q['mode']} {q['bits']}-bit g{q['group_size']}"
        kinds[key] = kinds.get(key, 0) + 1
    del model
    mx.clear_cache()

    reloaded, _ = load(out_dir)
    after = np.array(qlab.logits_for(reloaded, probe))
    max_diff = float(np.abs(before - after).max())
    files = sorted(f for f in os.listdir(out_dir) if f.endswith(".safetensors"))
    manifest = dict(
        name=name, recipe=recipe, upstream="thinkingmachines/Inkling-2-Mini", upstream_revision=UPSTREAM_REVISION,
        module_counts=kinds, reload_max_logit_diff=max_diff,
        weights_bytes=sum(os.path.getsize(os.path.join(out_dir, f)) for f in files if f != "mtp.safetensors"),
        files={f: dict(bytes=os.path.getsize(os.path.join(out_dir, f)), sha256=sha256(os.path.join(out_dir, f))) for f in files},
    )
    json.dump(manifest, open(os.path.join(out_dir, "build_manifest.json"), "w"), indent=1)
    print(json.dumps({k: v for k, v in manifest.items() if k not in ("files", "recipe")}, indent=1))
    if max_diff != 0.0:
        raise SystemExit(f"reloaded checkpoint differs from the in-memory model (max logit diff {max_diff})")


if __name__ == "__main__":
    main()