Instructions to use jbilcke/vllm-decision-2.0-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use jbilcke/vllm-decision-2.0-mlx with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download jbilcke/vllm-decision-2.0-mlx --local-dir vllm-decision-2.0-mlx
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download staged_source_reference.py from jbilcke/vllm-decision-2.0-mlx: direct link, hf CLI and curl.
- Browser
- Download file 22.2 kB
-
https://huggingface.co/jbilcke/vllm-decision-2.0-mlx/resolve/main/staged_source_reference.py
- Command line
-
hf download hf://jbilcke/vllm-decision-2.0-mlx/staged_source_reference.py
-
curl -L -o staged_source_reference.py https://huggingface.co/jbilcke/vllm-decision-2.0-mlx/resolve/main/staged_source_reference.py
22.2 kB
| """Experimental, CPU-only storage-dtype reference; importing this file allocates no model. | |
| This modifies upstream loading, not its trained tensors, prompt, readout or forward | |
| operators. Only backbone nn.Linear materializes temporary FP32 weights per call. | |
| It is NOT an unmodified upstream full-FP32 reference until smaller-model evidence | |
| establishes numerical equivalence. No network, MLX, MPS or CUDA path is used. | |
| Running requires --execute; Nox additionally requires three passing smaller-model | |
| prototype reports and an explicit peak-RSS limit. This is a memory research tool, | |
| not a guaranteed allocation bound (checks cannot prevent an individual allocation). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import importlib.metadata | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import platform | |
| import resource | |
| import struct | |
| import sys | |
| import time | |
| import types | |
| PINS = { | |
| "kai": ("vllm-sr/Decision-2.0-Kai-0.6B", "cd49ea3813fd8ba0928a9a23ef6c9a0f2f0cd764", "ca7bf2cff1494a5bfc51f4271f52557da0f3ce3b6f0f6e7a2b4118cb4965e92e", 8192, "qwen3"), | |
| "eos": ("vllm-sr/Decision-2.0-Eos-0.8B", "3594047d69f476f1d01cf84c593e213fc3a4dfe0", "e8b1081be4a76deca5247792a4031c8e19e2b52b95d91775c13d9e257c407101", 16384, "qwen3_5_text"), | |
| "sol": ("vllm-sr/Decision-2.0-Sol-2B", "64235bef55dad29387dd16da7c90e038bf2f0972", "a91c440ca371668d488a788bdfe12cc574a97123df60730afedf419737ced970", 16384, "qwen3_5_text"), | |
| "nox": ("vllm-sr/Decision-2.0-Nox-4B", "25e8f67d1b486c647222df3aac640d2d5d736bbe", "49771ea33a451274687ad2a246ce5fb78f8f42ec904fc91a467477179975bb16", 16384, "qwen3_5_text"), | |
| } | |
| METHOD = "experimental-source-storage-dtypes-cpu-fp32-linear-casts-v1" | |
| PROMPT = "decision2-segmented-options-global-query-v1" | |
| def canonical(value): | |
| return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) | |
| def file_sha(path): | |
| digest = hashlib.sha256() | |
| with Path(path).open("rb") as stream: | |
| for block in iter(lambda: stream.read(8 << 20), b""): | |
| digest.update(block) | |
| return digest.hexdigest() | |
| def peak_rss_bytes(): | |
| value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss | |
| return int(value if sys.platform == "darwin" else value * 1024) | |
| def check_rss(limit): | |
| peak = peak_rss_bytes() | |
| if limit and peak > limit: | |
| raise MemoryError(f"Whole-process peak RSS {peak} exceeds requested {limit}; no further model work") | |
| return peak | |
| def verify_source(source, variant): | |
| """Verify local source against an immutable Hub receipt before executing its code.""" | |
| source = Path(source).resolve() | |
| repo, revision, manifest_sha, cap, model_type = PINS[variant] | |
| receipt = json.loads((source / "HUB_METADATA.json").read_text()) | |
| if receipt.get("sha") != revision or receipt.get("id") != repo: | |
| raise ValueError("Wrong pinned Hub source identity") | |
| hashes = {} | |
| for entry in receipt["siblings"]: | |
| name = entry["rfilename"] | |
| if name.startswith("assets/") or name == ".gitattributes": | |
| continue | |
| relative = Path(name) | |
| if relative.is_absolute() or ".." in relative.parts: | |
| raise ValueError("Unsafe source receipt path") | |
| path = source / relative | |
| if path.is_symlink() or not path.is_file() or path.stat().st_size != entry["size"]: | |
| raise ValueError(f"Missing/different source file: {name}") | |
| sha = file_sha(path) | |
| if entry.get("lfs"): | |
| if sha != entry["lfs"]["sha256"]: | |
| raise ValueError(f"Source checksum failed: {name}") | |
| else: | |
| git = hashlib.sha1(f"blob {entry['size']}\0".encode()) | |
| with path.open("rb") as stream: | |
| for block in iter(lambda: stream.read(8 << 20), b""): | |
| git.update(block) | |
| if git.hexdigest() != entry["blobId"]: | |
| raise ValueError(f"Source Git checksum failed: {name}") | |
| hashes[name] = {"bytes": entry["size"], "sha256": sha} | |
| required = {"MODEL_MANIFEST.json", "backbone/config.json", "decision_config.json", "config.json", "tokenizer.json", "tokenizer_config.json", "decision_head.safetensors", "decision2/_vendor/dev2model/decision_model.py"} | |
| if not required <= hashes.keys(): | |
| raise ValueError("Incomplete receipt for staged reference") | |
| manifest = json.loads((source / "MODEL_MANIFEST.json").read_text()) | |
| if manifest_sha and hashes["MODEL_MANIFEST.json"]["sha256"] != manifest_sha: | |
| raise ValueError("Wrong pinned source manifest") | |
| # Pinned receipts/manifests detect local corruption and source drift; they | |
| # are not an independent publisher signature. | |
| for name, sha in manifest["files_sha256"].items(): | |
| if name.startswith("assets/"): | |
| continue | |
| if name not in hashes or hashes[name]["sha256"] != sha: | |
| raise ValueError(f"Original model manifest differs: {name}") | |
| backbone = json.loads((source / "backbone/config.json").read_text()) | |
| decision = json.loads((source / "decision_config.json").read_text()) | |
| wrapper = json.loads((source / "config.json").read_text()) | |
| if backbone.get("model_type") != model_type or wrapper.get("max_input_tokens") != cap: | |
| raise ValueError("Wrong pinned model architecture/context") | |
| if decision.get("head_variant", "shared") != "shared" or decision.get("head_dim") != 256 or decision.get("prompt_version") != PROMPT or decision.get("checkpoint_format", "full") != "full": | |
| raise ValueError("Only the released materialized shared-head checkpoint is supported") | |
| return source, hashes, backbone, decision, manifest | |
| def tensor_inventory(source): | |
| """Read only Safetensors JSON headers; retain source storage byte arithmetic.""" | |
| records = {} | |
| shards = sorted((source / "backbone").glob("*.safetensors")) + [source / "decision_head.safetensors"] | |
| if not shards: | |
| raise ValueError("No source shards") | |
| for shard in shards: | |
| with shard.open("rb") as stream: | |
| header_size = struct.unpack("<Q", stream.read(8))[0] | |
| if not 2 <= header_size <= 64 << 20: | |
| raise ValueError("Invalid Safetensors header length") | |
| header = json.loads(stream.read(header_size)) | |
| for key, item in header.items(): | |
| if key == "__metadata__": | |
| continue | |
| name = ("head." if shard.name == "decision_head.safetensors" else "backbone.") + key | |
| if name in records or item["dtype"] not in {"F32", "BF16"}: | |
| raise ValueError(f"Duplicate/unsupported source tensor: {name}") | |
| begin, end = item["data_offsets"] | |
| records[name] = {"shape": item["shape"], "dtype": item["dtype"], "storage_bytes": end - begin, "source_file": str(shard.relative_to(source)), "source_key": key} | |
| return records, shards | |
| def prepare_model(source, backbone_data, decision, inventory, shards, rss_limit): | |
| # Deferred imports: inspect/AST-parse this script without touching native runtimes. | |
| import torch | |
| import torch.nn.functional as functional | |
| from safetensors import safe_open | |
| from transformers import AutoTokenizer | |
| from transformers.models.qwen3.configuration_qwen3 import Qwen3Config | |
| from transformers.models.qwen3.modeling_qwen3 import Qwen3Model, Qwen3RotaryEmbedding | |
| from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig | |
| from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding | |
| if importlib.metadata.version("transformers") != "5.18.0": | |
| raise ValueError("Prototype requires source-reviewed Transformers 5.18.0") | |
| torch.set_num_threads(1) | |
| torch.set_default_dtype(torch.float32) | |
| sys.path.insert(0, str(source)) | |
| from decision2._vendor.dev2model.decision_model import CandidateHead, DecisionModel, encode, collate | |
| if backbone_data["model_type"] == "qwen3": | |
| config, model_class, rotary_class = Qwen3Config.from_dict(backbone_data), Qwen3Model, Qwen3RotaryEmbedding | |
| else: | |
| config, model_class, rotary_class = Qwen3_5TextConfig.from_dict(backbone_data), Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding | |
| config._attn_implementation = "sdpa" | |
| config.use_cache = False | |
| with torch.device("meta"): | |
| model = DecisionModel(model_class(config), CandidateHead(config.hidden_size, decision["head_dim"]), decision) | |
| expected = model.state_dict() | |
| if set(expected) != set(inventory): | |
| raise ValueError(f"Source/model tensor names differ: {sorted(set(expected) ^ set(inventory))}") | |
| for name, entry in inventory.items(): | |
| if list(expected[name].shape) != entry["shape"]: | |
| raise ValueError(f"Source tensor shape differs: {name}") | |
| del expected | |
| # Serial, mmap-backed assignments without a BF16→FP32 model-wide cast. | |
| for shard in shards: | |
| prefix = "head." if shard.name == "decision_head.safetensors" else "backbone." | |
| with safe_open(str(shard), framework="pt", device="cpu") as handle: | |
| for key in handle.keys(): | |
| name = prefix + key | |
| parent, attribute = name.rsplit(".", 1) | |
| module = model.get_submodule(parent) | |
| value = handle.get_tensor(key) | |
| if name.startswith("head.") and value.dtype != torch.float32: | |
| raise ValueError("Learned head must remain source FP32") | |
| if attribute in module._parameters: | |
| module._parameters[attribute] = torch.nn.Parameter(value, requires_grad=False) | |
| elif attribute in module._buffers: | |
| module._buffers[attribute] = value | |
| else: | |
| raise ValueError(f"Unexpected assignment target: {name}") | |
| del value | |
| check_rss(rss_limit) | |
| # Only nonpersistent RoPE is missing from state_dict. Recreate it through the | |
| # exact installed class on CPU, never by materializing all meta buffers. | |
| rope_count = 0 | |
| for module in model.backbone.modules(): | |
| if isinstance(module, rotary_class): | |
| if module.config.rope_parameters["rope_type"] != "default": | |
| raise ValueError("Prototype supports pinned default RoPE only") | |
| with torch.device("cpu"): | |
| replacement = rotary_class(config=module.config) | |
| module.inv_freq = torch.nn.Buffer(replacement.inv_freq, persistent=False) | |
| module.original_inv_freq = torch.nn.Buffer(replacement.original_inv_freq, persistent=False) | |
| module.attention_scaling = replacement.attention_scaling | |
| rope_count += 1 | |
| if rope_count != 1 or any(value.is_meta for value in list(model.parameters()) + list(model.buffers())): | |
| raise ValueError("Unmaterialized parameters/buffers after exact RoPE recreation") | |
| linear_count = 0 | |
| for name, module in model.backbone.named_modules(): | |
| if isinstance(module, torch.nn.Linear): | |
| def forward(layer, inputs): | |
| check_rss(rss_limit) | |
| # No persistent cast/cache. Temporary buffers die after F.linear; | |
| # the underlying allocator/mmap residency can still raise RSS. | |
| return functional.linear(inputs.float(), layer.weight.float(), None if layer.bias is None else layer.bias.float()) | |
| module.forward = types.MethodType(forward, module) | |
| linear_count += 1 | |
| else: | |
| for parameter in module.parameters(recurse=False): | |
| if parameter.dtype != torch.float32: | |
| raise ValueError(f"Nonlinear source parameter is not FP32: {name}") | |
| if not linear_count: | |
| raise ValueError("No backbone linear projections were wrapped") | |
| for layer in model.backbone.layers: | |
| layer.register_forward_hook(lambda module, arguments, output: check_rss(rss_limit) and None) | |
| model.eval() | |
| tokenizer = AutoTokenizer.from_pretrained(source, local_files_only=True, trust_remote_code=False) | |
| return model, tokenizer, encode, collate, linear_count | |
| def verify_smaller_proofs(paths): | |
| reports = [json.loads(Path(path).read_text()) for path in paths] | |
| if {report.get("variant") for report in reports} != {"kai", "eos", "sol"} or len(reports) != 3: | |
| raise ValueError("Nox requires one staged-vs-saved-FP32 proof for Kai, Eos and Sol") | |
| for report in reports: | |
| if report.get("method") != METHOD or report.get("verification", {}).get("status") != "passed" or report.get("source_revision") != PINS[report["variant"]][1]: | |
| raise ValueError("Unverified/wrong smaller-model staged reference proof") | |
| verification = report["verification"] | |
| if not report.get("reports") or verification.get("failed_fixtures") or verification.get("max_logit_abs_error", math.inf) > 1e-5 or verification.get("max_probability_abs_error", math.inf) > 1e-6 or verification.get("max_hidden_abs_error", math.inf) > 1e-5 or not all(row.get("saved_upstream_fp32_comparison", {}).get("passed") is True for row in report["reports"]): | |
| raise ValueError("Smaller-model proof lacks strict actual comparisons") | |
| def run(args): | |
| if not args.execute: | |
| raise ValueError("Prototype performs no model work without explicit --execute") | |
| if args.variant == "nox": | |
| if not args.max_rss_bytes or args.verify_against: | |
| raise ValueError("Nox requires a peak-RSS limit and cannot claim saved upstream FP32 parity") | |
| verify_smaller_proofs(args.smaller_proof) | |
| os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_TELEMETRY="1", HF_DEACTIVATE_ASYNC_LOAD="1") | |
| source, hashes, backbone, decision, manifest = verify_source(args.source, args.variant) | |
| inventory, shards = tensor_inventory(source) | |
| fixtures = json.loads(args.fixtures.read_text()) | |
| if not isinstance(fixtures, list) or not fixtures or len({row["id"] for row in fixtures}) != len(fixtures): | |
| raise ValueError("Nonempty unique-ID fixture list required") | |
| saved = None | |
| if args.verify_against: | |
| saved = json.loads(args.verify_against.read_text()) | |
| if saved.get("variant") != args.variant or saved.get("source_revision") != PINS[args.variant][1] or not saved.get("method", "").startswith("upstream CPU FP32 causal SDPA"): | |
| raise ValueError("Saved report is not the pinned actual CPU-FP32 reference") | |
| baseline = {row["id"]: row for row in saved["reports"] if row["precision"] == "float32-baseline"} | |
| if set(baseline) != {row["id"] for row in fixtures}: | |
| raise ValueError("Saved FP32 fixture IDs differ; do not silently drop cases") | |
| saved_readout = {row["id"]:row for row in saved.get("source_reference",[])} | |
| if set(saved_readout) != set(baseline): | |
| raise ValueError("Saved actual FP32 token/endpoint/hidden reference is required") | |
| rss_before = peak_rss_bytes() | |
| start = time.perf_counter() | |
| model, tokenizer, encode, collate, linear_count = prepare_model(source, backbone, decision, inventory, shards, args.max_rss_bytes) | |
| preparation_seconds = time.perf_counter() - start | |
| import torch | |
| observed = {} | |
| hook = model.backbone.register_forward_hook(lambda module,arguments,output:observed.update(hidden=output.last_hidden_state.detach())) | |
| results, failures = [], [] | |
| bias = json.loads((source / "score_bias.json").read_text()) if (source / "score_bias.json").exists() else None | |
| for fixture in fixtures: | |
| row = dict(fixture, label=0, family="synthetic-parity") | |
| encoded = encode(row, tokenizer, PINS[args.variant][3]) | |
| batch = collate([encoded], tokenizer.pad_token_id or tokenizer.eos_token_id) | |
| start = time.perf_counter() | |
| with torch.inference_mode(): | |
| raw = model(**batch)[0, :len(row["options"])].float().tolist() | |
| hidden=observed.pop("hidden") | |
| vectors=torch.cat([hidden[0,encoded["candidate_positions"]],hidden[0,encoded["query_position"]][None]],dim=0).float().tolist() | |
| del hidden | |
| if not all(math.isfinite(value) for value in raw): | |
| raise ValueError("Nonfinite staged candidate logits") | |
| logits = list(raw) | |
| if row["task_type"] == "score" and bias: | |
| offset = bias.get("offsets", {}).get(str(len(logits))) | |
| if offset: | |
| logits = [value + offset[int(option["key"])] for value, option in zip(logits, row["options"])] | |
| exponentials = [math.exp(value - max(logits)) for value in logits] | |
| probabilities = [value / sum(exponentials) for value in exponentials] | |
| record = {"id": row["id"], "ids": encoded["ids"], "candidate_positions": encoded["candidate_positions"], "query_position": encoded["query_position"], "token_ids_sha256": encoded["token_ids_sha256"], "prompt_sha256": encoded["prompt_sha256"], "keys": encoded["keys"], "raw_logits": raw, "score_bias_corrected_logits": logits, "probabilities": dict(zip(encoded["keys"], probabilities)), "cpu_seconds": time.perf_counter() - start, "whole_process_peak_rss_bytes": check_rss(args.max_rss_bytes)} | |
| if saved is not None: | |
| reference = baseline[row["id"]] | |
| readout = saved_readout[row["id"]] | |
| if any(readout[key] != encoded[key] for key in ("ids","candidate_positions","query_position","token_ids_sha256")): | |
| raise ValueError("Saved exact token/endpoint/query positions differ") | |
| if reference["token_ids_sha256"] != encoded["token_ids_sha256"] or reference["input_tokens"] != len(encoded["ids"]): | |
| raise ValueError("Saved reference token sequence differs") | |
| expected = reference["reference"] | |
| if len(expected["logits"]) != len(logits) or set(expected["probabilities"]) != set(encoded["keys"]): | |
| raise ValueError("Saved reference option menu differs") | |
| if not all(math.isfinite(value) for value in expected["logits"]) or not all(math.isfinite(value) for value in expected["probabilities"].values()): | |
| raise ValueError("Saved reference contains nonfinite scores") | |
| logit_error = max(abs(a - b) for a, b in zip(logits, expected["logits"])) | |
| probability_error = max(abs(p - expected["probabilities"][k]) for k, p in zip(encoded["keys"], probabilities)) | |
| passed = logit_error <= args.max_logit_error and probability_error <= args.max_probability_error | |
| if len(vectors)!=len(readout["hidden_readout"]) or any(len(a)!=len(b) for a,b in zip(vectors,readout["hidden_readout"])): | |
| raise ValueError("Saved endpoint/query hidden shapes differ") | |
| hidden_deltas=[a-b for first,second in zip(vectors,readout["hidden_readout"]) for a,b in zip(first,second)] | |
| hidden_max=max(abs(value) for value in hidden_deltas) | |
| hidden_rms=math.sqrt(sum(value*value for value in hidden_deltas)/len(hidden_deltas)) | |
| passed = passed and hidden_max <= args.max_hidden_error | |
| record["saved_upstream_fp32_comparison"] = {"max_logit_abs_error": logit_error, "max_probability_abs_error": probability_error, | |
| "hidden_readout_max_abs_error":hidden_max,"hidden_readout_rms_error":hidden_rms,"passed": passed} | |
| if not passed: | |
| failures.append(row["id"]) | |
| results.append(record) | |
| print("staged", row["id"], raw, flush=True) | |
| report = {"variant": args.variant, "source_repo": PINS[args.variant][0], "source_revision": PINS[args.variant][1], "method": METHOD, "upstream_loading_modified": True, "validation_level": "experimental-staged-source-reference", "source_hashes": hashes, "tensor_inventory": inventory, "source_storage_bytes": sum(value["storage_bytes"] for value in inventory.values()), "source_manifest_model_sha256": manifest["identity"]["model_sha256"], "prototype_sha256": file_sha(__file__), "fixtures_sha256": file_sha(args.fixtures), "platform": platform.platform(), "architecture": platform.machine(), "environment": {name: importlib.metadata.version(name) for name in ("torch", "transformers", "safetensors", "tokenizers")}, "backbone_linear_wrappers": linear_count, "preparation_seconds": preparation_seconds, "whole_process_peak_rss_before_bytes": rss_before, "whole_process_peak_rss_bytes": peak_rss_bytes(), "memory_caveat": "Whole-process peak RSS includes resident source mappings, allocator and activations; checks do not prevent an individual allocation or prove safe RAM use.", "verification": {"status": "not-compared" if saved is None else "failed" if failures else "passed", "saved_report_sha256": file_sha(args.verify_against) if saved is not None else None, "max_logit_abs_error": args.max_logit_error, "max_probability_abs_error": args.max_probability_error, "max_hidden_abs_error": args.max_hidden_error, "failed_fixtures": failures}, "reports": results} | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") | |
| hook.remove() | |
| if failures: | |
| raise ValueError(f"Staged reference differs from saved upstream FP32: {failures}; report retained") | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("variant", choices=PINS) | |
| parser.add_argument("--source", type=Path, required=True) | |
| parser.add_argument("--fixtures", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--execute", action="store_true") | |
| parser.add_argument("--verify-against", type=Path) | |
| parser.add_argument("--smaller-proof", type=Path, nargs="*", default=[]) | |
| parser.add_argument("--max-rss-bytes", type=int, default=0) | |
| parser.add_argument("--max-logit-error", type=float, default=1e-5) | |
| parser.add_argument("--max-probability-error", type=float, default=1e-6) | |
| parser.add_argument("--max-hidden-error", type=float, default=1e-5) | |
| args = parser.parse_args() | |
| if args.max_rss_bytes < 0 or not 0 < args.max_logit_error < math.inf or not 0 < args.max_probability_error < math.inf or not 0 < args.max_hidden_error < math.inf: | |
| parser.error("Invalid memory/numerical limits") | |
| run(args) | |
| if __name__ == "__main__": | |
| main() | |