on_the_way / serving /draft_model.py
RANITBAG's picture
2nd run
1a1fc1f verified
Raw History Blame Contribute Delete
3.09 kB
"""draft_model.py -- speculative-decoding draft compatibility interface.
What llama.cpp requires of a draft model (documented, verified only to the
extent the GGUF metadata allows):
* same architecture family (here: both are qwen3moe-host GGUFs produced by
inference/gguf/convert_rivinity_to_gguf.py) -- CHECKED from metadata
* same tokenizer (identical vocab size and tokenizer model) -- CHECKED
* smaller/faster than the target (advisory; cannot be verified from
metadata alone -- the launcher user must confirm the draft is small)
* the draft context must cover the speculation depth used -- ADVISORY
A Rivinity-Draft checkpoint (smaller Rivinity config, same tokenizer
artifact) satisfies this by construction. Using a third-party draft is
possible only if its GGUF reports the same architecture + tokenizer.
CLI: python serving/draft_model.py main.gguf draft.gguf
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from inference.gguf.convert_rivinity_to_gguf import ( # noqa: E402
ConversionError, read_gguf_header,
)
def check_draft_compatibility(main_gguf, draft_gguf) -> tuple:
"""Returns (problems, warnings). problems non-empty -> do not launch Turbo."""
problems, warnings = [], []
try:
mh = read_gguf_header(Path(main_gguf))
dh = read_gguf_header(Path(draft_gguf))
except (ConversionError, OSError, ValueError) as exc:
return [f"cannot read GGUF metadata: {exc}"], []
ma = mh["metadata"].get("general.architecture")
da = dh["metadata"].get("general.architecture")
if ma != da:
problems.append(f"architecture mismatch: main={ma!r} draft={da!r}")
mv = mh["metadata"].get("tokenizer.ggml.tokens", {}).get("count")
dv = dh["metadata"].get("tokenizer.ggml.tokens", {}).get("count")
if mv != dv:
problems.append(f"vocab size mismatch: main={mv} draft={dv} "
"(speculation requires identical token IDs)")
mm = mh["metadata"].get("tokenizer.ggml.model")
dm = dh["metadata"].get("tokenizer.ggml.model")
if mm != dm:
problems.append(f"tokenizer model mismatch: main={mm!r} draft={dm!r}")
if len(dh["tensors"]) >= len(mh["tensors"]):
warnings.append("draft has as many tensors as the target -- expected a "
"SMALLER model; verify the draft is actually faster")
return problems, warnings
def main(argv=None) -> int:
argv = sys.argv[1:] if argv is None else argv
if len(argv) != 2:
print(__doc__.splitlines()[0])
print("usage: python serving/draft_model.py main.gguf draft.gguf")
return 2
problems, warnings = check_draft_compatibility(argv[0], argv[1])
for w in warnings:
print(f"WARNING: {w}")
if problems:
for prob in problems:
print(f"INCOMPATIBLE: {prob}")
return 1
print("draft compatibility: OK (advisory checks still apply -- see docstring)")
return 0
if __name__ == "__main__":
raise SystemExit(main())