Instructions to use jbilcke/vllm-decision-2.0-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use jbilcke/vllm-decision-2.0-mlx with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download jbilcke/vllm-decision-2.0-mlx --local-dir vllm-decision-2.0-mlx
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 22,213 Bytes
49c548a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 | """Experimental, CPU-only storage-dtype reference; importing this file allocates no model.
This modifies upstream loading, not its trained tensors, prompt, readout or forward
operators. Only backbone nn.Linear materializes temporary FP32 weights per call.
It is NOT an unmodified upstream full-FP32 reference until smaller-model evidence
establishes numerical equivalence. No network, MLX, MPS or CUDA path is used.
Running requires --execute; Nox additionally requires three passing smaller-model
prototype reports and an explicit peak-RSS limit. This is a memory research tool,
not a guaranteed allocation bound (checks cannot prevent an individual allocation).
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.metadata
import json
import math
import os
from pathlib import Path
import platform
import resource
import struct
import sys
import time
import types
PINS = {
"kai": ("vllm-sr/Decision-2.0-Kai-0.6B", "cd49ea3813fd8ba0928a9a23ef6c9a0f2f0cd764", "ca7bf2cff1494a5bfc51f4271f52557da0f3ce3b6f0f6e7a2b4118cb4965e92e", 8192, "qwen3"),
"eos": ("vllm-sr/Decision-2.0-Eos-0.8B", "3594047d69f476f1d01cf84c593e213fc3a4dfe0", "e8b1081be4a76deca5247792a4031c8e19e2b52b95d91775c13d9e257c407101", 16384, "qwen3_5_text"),
"sol": ("vllm-sr/Decision-2.0-Sol-2B", "64235bef55dad29387dd16da7c90e038bf2f0972", "a91c440ca371668d488a788bdfe12cc574a97123df60730afedf419737ced970", 16384, "qwen3_5_text"),
"nox": ("vllm-sr/Decision-2.0-Nox-4B", "25e8f67d1b486c647222df3aac640d2d5d736bbe", "49771ea33a451274687ad2a246ce5fb78f8f42ec904fc91a467477179975bb16", 16384, "qwen3_5_text"),
}
METHOD = "experimental-source-storage-dtypes-cpu-fp32-linear-casts-v1"
PROMPT = "decision2-segmented-options-global-query-v1"
def canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
def file_sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(8 << 20), b""):
digest.update(block)
return digest.hexdigest()
def peak_rss_bytes():
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return int(value if sys.platform == "darwin" else value * 1024)
def check_rss(limit):
peak = peak_rss_bytes()
if limit and peak > limit:
raise MemoryError(f"Whole-process peak RSS {peak} exceeds requested {limit}; no further model work")
return peak
def verify_source(source, variant):
"""Verify local source against an immutable Hub receipt before executing its code."""
source = Path(source).resolve()
repo, revision, manifest_sha, cap, model_type = PINS[variant]
receipt = json.loads((source / "HUB_METADATA.json").read_text())
if receipt.get("sha") != revision or receipt.get("id") != repo:
raise ValueError("Wrong pinned Hub source identity")
hashes = {}
for entry in receipt["siblings"]:
name = entry["rfilename"]
if name.startswith("assets/") or name == ".gitattributes":
continue
relative = Path(name)
if relative.is_absolute() or ".." in relative.parts:
raise ValueError("Unsafe source receipt path")
path = source / relative
if path.is_symlink() or not path.is_file() or path.stat().st_size != entry["size"]:
raise ValueError(f"Missing/different source file: {name}")
sha = file_sha(path)
if entry.get("lfs"):
if sha != entry["lfs"]["sha256"]:
raise ValueError(f"Source checksum failed: {name}")
else:
git = hashlib.sha1(f"blob {entry['size']}\0".encode())
with path.open("rb") as stream:
for block in iter(lambda: stream.read(8 << 20), b""):
git.update(block)
if git.hexdigest() != entry["blobId"]:
raise ValueError(f"Source Git checksum failed: {name}")
hashes[name] = {"bytes": entry["size"], "sha256": sha}
required = {"MODEL_MANIFEST.json", "backbone/config.json", "decision_config.json", "config.json", "tokenizer.json", "tokenizer_config.json", "decision_head.safetensors", "decision2/_vendor/dev2model/decision_model.py"}
if not required <= hashes.keys():
raise ValueError("Incomplete receipt for staged reference")
manifest = json.loads((source / "MODEL_MANIFEST.json").read_text())
if manifest_sha and hashes["MODEL_MANIFEST.json"]["sha256"] != manifest_sha:
raise ValueError("Wrong pinned source manifest")
# Pinned receipts/manifests detect local corruption and source drift; they
# are not an independent publisher signature.
for name, sha in manifest["files_sha256"].items():
if name.startswith("assets/"):
continue
if name not in hashes or hashes[name]["sha256"] != sha:
raise ValueError(f"Original model manifest differs: {name}")
backbone = json.loads((source / "backbone/config.json").read_text())
decision = json.loads((source / "decision_config.json").read_text())
wrapper = json.loads((source / "config.json").read_text())
if backbone.get("model_type") != model_type or wrapper.get("max_input_tokens") != cap:
raise ValueError("Wrong pinned model architecture/context")
if decision.get("head_variant", "shared") != "shared" or decision.get("head_dim") != 256 or decision.get("prompt_version") != PROMPT or decision.get("checkpoint_format", "full") != "full":
raise ValueError("Only the released materialized shared-head checkpoint is supported")
return source, hashes, backbone, decision, manifest
def tensor_inventory(source):
"""Read only Safetensors JSON headers; retain source storage byte arithmetic."""
records = {}
shards = sorted((source / "backbone").glob("*.safetensors")) + [source / "decision_head.safetensors"]
if not shards:
raise ValueError("No source shards")
for shard in shards:
with shard.open("rb") as stream:
header_size = struct.unpack("<Q", stream.read(8))[0]
if not 2 <= header_size <= 64 << 20:
raise ValueError("Invalid Safetensors header length")
header = json.loads(stream.read(header_size))
for key, item in header.items():
if key == "__metadata__":
continue
name = ("head." if shard.name == "decision_head.safetensors" else "backbone.") + key
if name in records or item["dtype"] not in {"F32", "BF16"}:
raise ValueError(f"Duplicate/unsupported source tensor: {name}")
begin, end = item["data_offsets"]
records[name] = {"shape": item["shape"], "dtype": item["dtype"], "storage_bytes": end - begin, "source_file": str(shard.relative_to(source)), "source_key": key}
return records, shards
def prepare_model(source, backbone_data, decision, inventory, shards, rss_limit):
# Deferred imports: inspect/AST-parse this script without touching native runtimes.
import torch
import torch.nn.functional as functional
from safetensors import safe_open
from transformers import AutoTokenizer
from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
from transformers.models.qwen3.modeling_qwen3 import Qwen3Model, Qwen3RotaryEmbedding
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding
if importlib.metadata.version("transformers") != "5.18.0":
raise ValueError("Prototype requires source-reviewed Transformers 5.18.0")
torch.set_num_threads(1)
torch.set_default_dtype(torch.float32)
sys.path.insert(0, str(source))
from decision2._vendor.dev2model.decision_model import CandidateHead, DecisionModel, encode, collate
if backbone_data["model_type"] == "qwen3":
config, model_class, rotary_class = Qwen3Config.from_dict(backbone_data), Qwen3Model, Qwen3RotaryEmbedding
else:
config, model_class, rotary_class = Qwen3_5TextConfig.from_dict(backbone_data), Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding
config._attn_implementation = "sdpa"
config.use_cache = False
with torch.device("meta"):
model = DecisionModel(model_class(config), CandidateHead(config.hidden_size, decision["head_dim"]), decision)
expected = model.state_dict()
if set(expected) != set(inventory):
raise ValueError(f"Source/model tensor names differ: {sorted(set(expected) ^ set(inventory))}")
for name, entry in inventory.items():
if list(expected[name].shape) != entry["shape"]:
raise ValueError(f"Source tensor shape differs: {name}")
del expected
# Serial, mmap-backed assignments without a BF16→FP32 model-wide cast.
for shard in shards:
prefix = "head." if shard.name == "decision_head.safetensors" else "backbone."
with safe_open(str(shard), framework="pt", device="cpu") as handle:
for key in handle.keys():
name = prefix + key
parent, attribute = name.rsplit(".", 1)
module = model.get_submodule(parent)
value = handle.get_tensor(key)
if name.startswith("head.") and value.dtype != torch.float32:
raise ValueError("Learned head must remain source FP32")
if attribute in module._parameters:
module._parameters[attribute] = torch.nn.Parameter(value, requires_grad=False)
elif attribute in module._buffers:
module._buffers[attribute] = value
else:
raise ValueError(f"Unexpected assignment target: {name}")
del value
check_rss(rss_limit)
# Only nonpersistent RoPE is missing from state_dict. Recreate it through the
# exact installed class on CPU, never by materializing all meta buffers.
rope_count = 0
for module in model.backbone.modules():
if isinstance(module, rotary_class):
if module.config.rope_parameters["rope_type"] != "default":
raise ValueError("Prototype supports pinned default RoPE only")
with torch.device("cpu"):
replacement = rotary_class(config=module.config)
module.inv_freq = torch.nn.Buffer(replacement.inv_freq, persistent=False)
module.original_inv_freq = torch.nn.Buffer(replacement.original_inv_freq, persistent=False)
module.attention_scaling = replacement.attention_scaling
rope_count += 1
if rope_count != 1 or any(value.is_meta for value in list(model.parameters()) + list(model.buffers())):
raise ValueError("Unmaterialized parameters/buffers after exact RoPE recreation")
linear_count = 0
for name, module in model.backbone.named_modules():
if isinstance(module, torch.nn.Linear):
def forward(layer, inputs):
check_rss(rss_limit)
# No persistent cast/cache. Temporary buffers die after F.linear;
# the underlying allocator/mmap residency can still raise RSS.
return functional.linear(inputs.float(), layer.weight.float(), None if layer.bias is None else layer.bias.float())
module.forward = types.MethodType(forward, module)
linear_count += 1
else:
for parameter in module.parameters(recurse=False):
if parameter.dtype != torch.float32:
raise ValueError(f"Nonlinear source parameter is not FP32: {name}")
if not linear_count:
raise ValueError("No backbone linear projections were wrapped")
for layer in model.backbone.layers:
layer.register_forward_hook(lambda module, arguments, output: check_rss(rss_limit) and None)
model.eval()
tokenizer = AutoTokenizer.from_pretrained(source, local_files_only=True, trust_remote_code=False)
return model, tokenizer, encode, collate, linear_count
def verify_smaller_proofs(paths):
reports = [json.loads(Path(path).read_text()) for path in paths]
if {report.get("variant") for report in reports} != {"kai", "eos", "sol"} or len(reports) != 3:
raise ValueError("Nox requires one staged-vs-saved-FP32 proof for Kai, Eos and Sol")
for report in reports:
if report.get("method") != METHOD or report.get("verification", {}).get("status") != "passed" or report.get("source_revision") != PINS[report["variant"]][1]:
raise ValueError("Unverified/wrong smaller-model staged reference proof")
verification = report["verification"]
if not report.get("reports") or verification.get("failed_fixtures") or verification.get("max_logit_abs_error", math.inf) > 1e-5 or verification.get("max_probability_abs_error", math.inf) > 1e-6 or verification.get("max_hidden_abs_error", math.inf) > 1e-5 or not all(row.get("saved_upstream_fp32_comparison", {}).get("passed") is True for row in report["reports"]):
raise ValueError("Smaller-model proof lacks strict actual comparisons")
def run(args):
if not args.execute:
raise ValueError("Prototype performs no model work without explicit --execute")
if args.variant == "nox":
if not args.max_rss_bytes or args.verify_against:
raise ValueError("Nox requires a peak-RSS limit and cannot claim saved upstream FP32 parity")
verify_smaller_proofs(args.smaller_proof)
os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_TELEMETRY="1", HF_DEACTIVATE_ASYNC_LOAD="1")
source, hashes, backbone, decision, manifest = verify_source(args.source, args.variant)
inventory, shards = tensor_inventory(source)
fixtures = json.loads(args.fixtures.read_text())
if not isinstance(fixtures, list) or not fixtures or len({row["id"] for row in fixtures}) != len(fixtures):
raise ValueError("Nonempty unique-ID fixture list required")
saved = None
if args.verify_against:
saved = json.loads(args.verify_against.read_text())
if saved.get("variant") != args.variant or saved.get("source_revision") != PINS[args.variant][1] or not saved.get("method", "").startswith("upstream CPU FP32 causal SDPA"):
raise ValueError("Saved report is not the pinned actual CPU-FP32 reference")
baseline = {row["id"]: row for row in saved["reports"] if row["precision"] == "float32-baseline"}
if set(baseline) != {row["id"] for row in fixtures}:
raise ValueError("Saved FP32 fixture IDs differ; do not silently drop cases")
saved_readout = {row["id"]:row for row in saved.get("source_reference",[])}
if set(saved_readout) != set(baseline):
raise ValueError("Saved actual FP32 token/endpoint/hidden reference is required")
rss_before = peak_rss_bytes()
start = time.perf_counter()
model, tokenizer, encode, collate, linear_count = prepare_model(source, backbone, decision, inventory, shards, args.max_rss_bytes)
preparation_seconds = time.perf_counter() - start
import torch
observed = {}
hook = model.backbone.register_forward_hook(lambda module,arguments,output:observed.update(hidden=output.last_hidden_state.detach()))
results, failures = [], []
bias = json.loads((source / "score_bias.json").read_text()) if (source / "score_bias.json").exists() else None
for fixture in fixtures:
row = dict(fixture, label=0, family="synthetic-parity")
encoded = encode(row, tokenizer, PINS[args.variant][3])
batch = collate([encoded], tokenizer.pad_token_id or tokenizer.eos_token_id)
start = time.perf_counter()
with torch.inference_mode():
raw = model(**batch)[0, :len(row["options"])].float().tolist()
hidden=observed.pop("hidden")
vectors=torch.cat([hidden[0,encoded["candidate_positions"]],hidden[0,encoded["query_position"]][None]],dim=0).float().tolist()
del hidden
if not all(math.isfinite(value) for value in raw):
raise ValueError("Nonfinite staged candidate logits")
logits = list(raw)
if row["task_type"] == "score" and bias:
offset = bias.get("offsets", {}).get(str(len(logits)))
if offset:
logits = [value + offset[int(option["key"])] for value, option in zip(logits, row["options"])]
exponentials = [math.exp(value - max(logits)) for value in logits]
probabilities = [value / sum(exponentials) for value in exponentials]
record = {"id": row["id"], "ids": encoded["ids"], "candidate_positions": encoded["candidate_positions"], "query_position": encoded["query_position"], "token_ids_sha256": encoded["token_ids_sha256"], "prompt_sha256": encoded["prompt_sha256"], "keys": encoded["keys"], "raw_logits": raw, "score_bias_corrected_logits": logits, "probabilities": dict(zip(encoded["keys"], probabilities)), "cpu_seconds": time.perf_counter() - start, "whole_process_peak_rss_bytes": check_rss(args.max_rss_bytes)}
if saved is not None:
reference = baseline[row["id"]]
readout = saved_readout[row["id"]]
if any(readout[key] != encoded[key] for key in ("ids","candidate_positions","query_position","token_ids_sha256")):
raise ValueError("Saved exact token/endpoint/query positions differ")
if reference["token_ids_sha256"] != encoded["token_ids_sha256"] or reference["input_tokens"] != len(encoded["ids"]):
raise ValueError("Saved reference token sequence differs")
expected = reference["reference"]
if len(expected["logits"]) != len(logits) or set(expected["probabilities"]) != set(encoded["keys"]):
raise ValueError("Saved reference option menu differs")
if not all(math.isfinite(value) for value in expected["logits"]) or not all(math.isfinite(value) for value in expected["probabilities"].values()):
raise ValueError("Saved reference contains nonfinite scores")
logit_error = max(abs(a - b) for a, b in zip(logits, expected["logits"]))
probability_error = max(abs(p - expected["probabilities"][k]) for k, p in zip(encoded["keys"], probabilities))
passed = logit_error <= args.max_logit_error and probability_error <= args.max_probability_error
if len(vectors)!=len(readout["hidden_readout"]) or any(len(a)!=len(b) for a,b in zip(vectors,readout["hidden_readout"])):
raise ValueError("Saved endpoint/query hidden shapes differ")
hidden_deltas=[a-b for first,second in zip(vectors,readout["hidden_readout"]) for a,b in zip(first,second)]
hidden_max=max(abs(value) for value in hidden_deltas)
hidden_rms=math.sqrt(sum(value*value for value in hidden_deltas)/len(hidden_deltas))
passed = passed and hidden_max <= args.max_hidden_error
record["saved_upstream_fp32_comparison"] = {"max_logit_abs_error": logit_error, "max_probability_abs_error": probability_error,
"hidden_readout_max_abs_error":hidden_max,"hidden_readout_rms_error":hidden_rms,"passed": passed}
if not passed:
failures.append(row["id"])
results.append(record)
print("staged", row["id"], raw, flush=True)
report = {"variant": args.variant, "source_repo": PINS[args.variant][0], "source_revision": PINS[args.variant][1], "method": METHOD, "upstream_loading_modified": True, "validation_level": "experimental-staged-source-reference", "source_hashes": hashes, "tensor_inventory": inventory, "source_storage_bytes": sum(value["storage_bytes"] for value in inventory.values()), "source_manifest_model_sha256": manifest["identity"]["model_sha256"], "prototype_sha256": file_sha(__file__), "fixtures_sha256": file_sha(args.fixtures), "platform": platform.platform(), "architecture": platform.machine(), "environment": {name: importlib.metadata.version(name) for name in ("torch", "transformers", "safetensors", "tokenizers")}, "backbone_linear_wrappers": linear_count, "preparation_seconds": preparation_seconds, "whole_process_peak_rss_before_bytes": rss_before, "whole_process_peak_rss_bytes": peak_rss_bytes(), "memory_caveat": "Whole-process peak RSS includes resident source mappings, allocator and activations; checks do not prevent an individual allocation or prove safe RAM use.", "verification": {"status": "not-compared" if saved is None else "failed" if failures else "passed", "saved_report_sha256": file_sha(args.verify_against) if saved is not None else None, "max_logit_abs_error": args.max_logit_error, "max_probability_abs_error": args.max_probability_error, "max_hidden_abs_error": args.max_hidden_error, "failed_fixtures": failures}, "reports": results}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
hook.remove()
if failures:
raise ValueError(f"Staged reference differs from saved upstream FP32: {failures}; report retained")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("variant", choices=PINS)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--fixtures", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--verify-against", type=Path)
parser.add_argument("--smaller-proof", type=Path, nargs="*", default=[])
parser.add_argument("--max-rss-bytes", type=int, default=0)
parser.add_argument("--max-logit-error", type=float, default=1e-5)
parser.add_argument("--max-probability-error", type=float, default=1e-6)
parser.add_argument("--max-hidden-error", type=float, default=1e-5)
args = parser.parse_args()
if args.max_rss_bytes < 0 or not 0 < args.max_logit_error < math.inf or not 0 < args.max_probability_error < math.inf or not 0 < args.max_hidden_error < math.inf:
parser.error("Invalid memory/numerical limits")
run(args)
if __name__ == "__main__":
main()
|