Download serving/draft_model.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 3.09 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/draft_model.py
- Command line
-
hf download hf://RANITBAG/on_the_way/serving/draft_model.py
-
curl -L -o draft_model.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/draft_model.py
3.09 kB
| """draft_model.py -- speculative-decoding draft compatibility interface. | |
| What llama.cpp requires of a draft model (documented, verified only to the | |
| extent the GGUF metadata allows): | |
| * same architecture family (here: both are qwen3moe-host GGUFs produced by | |
| inference/gguf/convert_rivinity_to_gguf.py) -- CHECKED from metadata | |
| * same tokenizer (identical vocab size and tokenizer model) -- CHECKED | |
| * smaller/faster than the target (advisory; cannot be verified from | |
| metadata alone -- the launcher user must confirm the draft is small) | |
| * the draft context must cover the speculation depth used -- ADVISORY | |
| A Rivinity-Draft checkpoint (smaller Rivinity config, same tokenizer | |
| artifact) satisfies this by construction. Using a third-party draft is | |
| possible only if its GGUF reports the same architecture + tokenizer. | |
| CLI: python serving/draft_model.py main.gguf draft.gguf | |
| """ | |
| from __future__ import annotations | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from inference.gguf.convert_rivinity_to_gguf import ( # noqa: E402 | |
| ConversionError, read_gguf_header, | |
| ) | |
| def check_draft_compatibility(main_gguf, draft_gguf) -> tuple: | |
| """Returns (problems, warnings). problems non-empty -> do not launch Turbo.""" | |
| problems, warnings = [], [] | |
| try: | |
| mh = read_gguf_header(Path(main_gguf)) | |
| dh = read_gguf_header(Path(draft_gguf)) | |
| except (ConversionError, OSError, ValueError) as exc: | |
| return [f"cannot read GGUF metadata: {exc}"], [] | |
| ma = mh["metadata"].get("general.architecture") | |
| da = dh["metadata"].get("general.architecture") | |
| if ma != da: | |
| problems.append(f"architecture mismatch: main={ma!r} draft={da!r}") | |
| mv = mh["metadata"].get("tokenizer.ggml.tokens", {}).get("count") | |
| dv = dh["metadata"].get("tokenizer.ggml.tokens", {}).get("count") | |
| if mv != dv: | |
| problems.append(f"vocab size mismatch: main={mv} draft={dv} " | |
| "(speculation requires identical token IDs)") | |
| mm = mh["metadata"].get("tokenizer.ggml.model") | |
| dm = dh["metadata"].get("tokenizer.ggml.model") | |
| if mm != dm: | |
| problems.append(f"tokenizer model mismatch: main={mm!r} draft={dm!r}") | |
| if len(dh["tensors"]) >= len(mh["tensors"]): | |
| warnings.append("draft has as many tensors as the target -- expected a " | |
| "SMALLER model; verify the draft is actually faster") | |
| return problems, warnings | |
| def main(argv=None) -> int: | |
| argv = sys.argv[1:] if argv is None else argv | |
| if len(argv) != 2: | |
| print(__doc__.splitlines()[0]) | |
| print("usage: python serving/draft_model.py main.gguf draft.gguf") | |
| return 2 | |
| problems, warnings = check_draft_compatibility(argv[0], argv[1]) | |
| for w in warnings: | |
| print(f"WARNING: {w}") | |
| if problems: | |
| for prob in problems: | |
| print(f"INCOMPATIBLE: {prob}") | |
| return 1 | |
| print("draft compatibility: OK (advisory checks still apply -- see docstring)") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |