"""draft_model.py -- speculative-decoding draft compatibility interface. What llama.cpp requires of a draft model (documented, verified only to the extent the GGUF metadata allows): * same architecture family (here: both are qwen3moe-host GGUFs produced by inference/gguf/convert_rivinity_to_gguf.py) -- CHECKED from metadata * same tokenizer (identical vocab size and tokenizer model) -- CHECKED * smaller/faster than the target (advisory; cannot be verified from metadata alone -- the launcher user must confirm the draft is small) * the draft context must cover the speculation depth used -- ADVISORY A Rivinity-Draft checkpoint (smaller Rivinity config, same tokenizer artifact) satisfies this by construction. Using a third-party draft is possible only if its GGUF reports the same architecture + tokenizer. CLI: python serving/draft_model.py main.gguf draft.gguf """ from __future__ import annotations import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from inference.gguf.convert_rivinity_to_gguf import ( # noqa: E402 ConversionError, read_gguf_header, ) def check_draft_compatibility(main_gguf, draft_gguf) -> tuple: """Returns (problems, warnings). problems non-empty -> do not launch Turbo.""" problems, warnings = [], [] try: mh = read_gguf_header(Path(main_gguf)) dh = read_gguf_header(Path(draft_gguf)) except (ConversionError, OSError, ValueError) as exc: return [f"cannot read GGUF metadata: {exc}"], [] ma = mh["metadata"].get("general.architecture") da = dh["metadata"].get("general.architecture") if ma != da: problems.append(f"architecture mismatch: main={ma!r} draft={da!r}") mv = mh["metadata"].get("tokenizer.ggml.tokens", {}).get("count") dv = dh["metadata"].get("tokenizer.ggml.tokens", {}).get("count") if mv != dv: problems.append(f"vocab size mismatch: main={mv} draft={dv} " "(speculation requires identical token IDs)") mm = mh["metadata"].get("tokenizer.ggml.model") dm = dh["metadata"].get("tokenizer.ggml.model") if mm != dm: problems.append(f"tokenizer model mismatch: main={mm!r} draft={dm!r}") if len(dh["tensors"]) >= len(mh["tensors"]): warnings.append("draft has as many tensors as the target -- expected a " "SMALLER model; verify the draft is actually faster") return problems, warnings def main(argv=None) -> int: argv = sys.argv[1:] if argv is None else argv if len(argv) != 2: print(__doc__.splitlines()[0]) print("usage: python serving/draft_model.py main.gguf draft.gguf") return 2 problems, warnings = check_draft_compatibility(argv[0], argv[1]) for w in warnings: print(f"WARNING: {w}") if problems: for prob in problems: print(f"INCOMPATIBLE: {prob}") return 1 print("draft compatibility: OK (advisory checks still apply -- see docstring)") return 0 if __name__ == "__main__": raise SystemExit(main())