vllm-decision-2.0-mlx / staged_source_reference.py
jbilcke's picture
Publish validated Decision 2.0 MLX family
49c548a verified
Raw History Blame Contribute Delete
22.2 kB
"""Experimental, CPU-only storage-dtype reference; importing this file allocates no model.
This modifies upstream loading, not its trained tensors, prompt, readout or forward
operators. Only backbone nn.Linear materializes temporary FP32 weights per call.
It is NOT an unmodified upstream full-FP32 reference until smaller-model evidence
establishes numerical equivalence. No network, MLX, MPS or CUDA path is used.
Running requires --execute; Nox additionally requires three passing smaller-model
prototype reports and an explicit peak-RSS limit. This is a memory research tool,
not a guaranteed allocation bound (checks cannot prevent an individual allocation).
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.metadata
import json
import math
import os
from pathlib import Path
import platform
import resource
import struct
import sys
import time
import types
PINS = {
"kai": ("vllm-sr/Decision-2.0-Kai-0.6B", "cd49ea3813fd8ba0928a9a23ef6c9a0f2f0cd764", "ca7bf2cff1494a5bfc51f4271f52557da0f3ce3b6f0f6e7a2b4118cb4965e92e", 8192, "qwen3"),
"eos": ("vllm-sr/Decision-2.0-Eos-0.8B", "3594047d69f476f1d01cf84c593e213fc3a4dfe0", "e8b1081be4a76deca5247792a4031c8e19e2b52b95d91775c13d9e257c407101", 16384, "qwen3_5_text"),
"sol": ("vllm-sr/Decision-2.0-Sol-2B", "64235bef55dad29387dd16da7c90e038bf2f0972", "a91c440ca371668d488a788bdfe12cc574a97123df60730afedf419737ced970", 16384, "qwen3_5_text"),
"nox": ("vllm-sr/Decision-2.0-Nox-4B", "25e8f67d1b486c647222df3aac640d2d5d736bbe", "49771ea33a451274687ad2a246ce5fb78f8f42ec904fc91a467477179975bb16", 16384, "qwen3_5_text"),
}
METHOD = "experimental-source-storage-dtypes-cpu-fp32-linear-casts-v1"
PROMPT = "decision2-segmented-options-global-query-v1"
def canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
def file_sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(8 << 20), b""):
digest.update(block)
return digest.hexdigest()
def peak_rss_bytes():
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return int(value if sys.platform == "darwin" else value * 1024)
def check_rss(limit):
peak = peak_rss_bytes()
if limit and peak > limit:
raise MemoryError(f"Whole-process peak RSS {peak} exceeds requested {limit}; no further model work")
return peak
def verify_source(source, variant):
"""Verify local source against an immutable Hub receipt before executing its code."""
source = Path(source).resolve()
repo, revision, manifest_sha, cap, model_type = PINS[variant]
receipt = json.loads((source / "HUB_METADATA.json").read_text())
if receipt.get("sha") != revision or receipt.get("id") != repo:
raise ValueError("Wrong pinned Hub source identity")
hashes = {}
for entry in receipt["siblings"]:
name = entry["rfilename"]
if name.startswith("assets/") or name == ".gitattributes":
continue
relative = Path(name)
if relative.is_absolute() or ".." in relative.parts:
raise ValueError("Unsafe source receipt path")
path = source / relative
if path.is_symlink() or not path.is_file() or path.stat().st_size != entry["size"]:
raise ValueError(f"Missing/different source file: {name}")
sha = file_sha(path)
if entry.get("lfs"):
if sha != entry["lfs"]["sha256"]:
raise ValueError(f"Source checksum failed: {name}")
else:
git = hashlib.sha1(f"blob {entry['size']}\0".encode())
with path.open("rb") as stream:
for block in iter(lambda: stream.read(8 << 20), b""):
git.update(block)
if git.hexdigest() != entry["blobId"]:
raise ValueError(f"Source Git checksum failed: {name}")
hashes[name] = {"bytes": entry["size"], "sha256": sha}
required = {"MODEL_MANIFEST.json", "backbone/config.json", "decision_config.json", "config.json", "tokenizer.json", "tokenizer_config.json", "decision_head.safetensors", "decision2/_vendor/dev2model/decision_model.py"}
if not required <= hashes.keys():
raise ValueError("Incomplete receipt for staged reference")
manifest = json.loads((source / "MODEL_MANIFEST.json").read_text())
if manifest_sha and hashes["MODEL_MANIFEST.json"]["sha256"] != manifest_sha:
raise ValueError("Wrong pinned source manifest")
# Pinned receipts/manifests detect local corruption and source drift; they
# are not an independent publisher signature.
for name, sha in manifest["files_sha256"].items():
if name.startswith("assets/"):
continue
if name not in hashes or hashes[name]["sha256"] != sha:
raise ValueError(f"Original model manifest differs: {name}")
backbone = json.loads((source / "backbone/config.json").read_text())
decision = json.loads((source / "decision_config.json").read_text())
wrapper = json.loads((source / "config.json").read_text())
if backbone.get("model_type") != model_type or wrapper.get("max_input_tokens") != cap:
raise ValueError("Wrong pinned model architecture/context")
if decision.get("head_variant", "shared") != "shared" or decision.get("head_dim") != 256 or decision.get("prompt_version") != PROMPT or decision.get("checkpoint_format", "full") != "full":
raise ValueError("Only the released materialized shared-head checkpoint is supported")
return source, hashes, backbone, decision, manifest
def tensor_inventory(source):
"""Read only Safetensors JSON headers; retain source storage byte arithmetic."""
records = {}
shards = sorted((source / "backbone").glob("*.safetensors")) + [source / "decision_head.safetensors"]
if not shards:
raise ValueError("No source shards")
for shard in shards:
with shard.open("rb") as stream:
header_size = struct.unpack("<Q", stream.read(8))[0]
if not 2 <= header_size <= 64 << 20:
raise ValueError("Invalid Safetensors header length")
header = json.loads(stream.read(header_size))
for key, item in header.items():
if key == "__metadata__":
continue
name = ("head." if shard.name == "decision_head.safetensors" else "backbone.") + key
if name in records or item["dtype"] not in {"F32", "BF16"}:
raise ValueError(f"Duplicate/unsupported source tensor: {name}")
begin, end = item["data_offsets"]
records[name] = {"shape": item["shape"], "dtype": item["dtype"], "storage_bytes": end - begin, "source_file": str(shard.relative_to(source)), "source_key": key}
return records, shards
def prepare_model(source, backbone_data, decision, inventory, shards, rss_limit):
# Deferred imports: inspect/AST-parse this script without touching native runtimes.
import torch
import torch.nn.functional as functional
from safetensors import safe_open
from transformers import AutoTokenizer
from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
from transformers.models.qwen3.modeling_qwen3 import Qwen3Model, Qwen3RotaryEmbedding
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding
if importlib.metadata.version("transformers") != "5.18.0":
raise ValueError("Prototype requires source-reviewed Transformers 5.18.0")
torch.set_num_threads(1)
torch.set_default_dtype(torch.float32)
sys.path.insert(0, str(source))
from decision2._vendor.dev2model.decision_model import CandidateHead, DecisionModel, encode, collate
if backbone_data["model_type"] == "qwen3":
config, model_class, rotary_class = Qwen3Config.from_dict(backbone_data), Qwen3Model, Qwen3RotaryEmbedding
else:
config, model_class, rotary_class = Qwen3_5TextConfig.from_dict(backbone_data), Qwen3_5TextModel, Qwen3_5TextRotaryEmbedding
config._attn_implementation = "sdpa"
config.use_cache = False
with torch.device("meta"):
model = DecisionModel(model_class(config), CandidateHead(config.hidden_size, decision["head_dim"]), decision)
expected = model.state_dict()
if set(expected) != set(inventory):
raise ValueError(f"Source/model tensor names differ: {sorted(set(expected) ^ set(inventory))}")
for name, entry in inventory.items():
if list(expected[name].shape) != entry["shape"]:
raise ValueError(f"Source tensor shape differs: {name}")
del expected
# Serial, mmap-backed assignments without a BF16→FP32 model-wide cast.
for shard in shards:
prefix = "head." if shard.name == "decision_head.safetensors" else "backbone."
with safe_open(str(shard), framework="pt", device="cpu") as handle:
for key in handle.keys():
name = prefix + key
parent, attribute = name.rsplit(".", 1)
module = model.get_submodule(parent)
value = handle.get_tensor(key)
if name.startswith("head.") and value.dtype != torch.float32:
raise ValueError("Learned head must remain source FP32")
if attribute in module._parameters:
module._parameters[attribute] = torch.nn.Parameter(value, requires_grad=False)
elif attribute in module._buffers:
module._buffers[attribute] = value
else:
raise ValueError(f"Unexpected assignment target: {name}")
del value
check_rss(rss_limit)
# Only nonpersistent RoPE is missing from state_dict. Recreate it through the
# exact installed class on CPU, never by materializing all meta buffers.
rope_count = 0
for module in model.backbone.modules():
if isinstance(module, rotary_class):
if module.config.rope_parameters["rope_type"] != "default":
raise ValueError("Prototype supports pinned default RoPE only")
with torch.device("cpu"):
replacement = rotary_class(config=module.config)
module.inv_freq = torch.nn.Buffer(replacement.inv_freq, persistent=False)
module.original_inv_freq = torch.nn.Buffer(replacement.original_inv_freq, persistent=False)
module.attention_scaling = replacement.attention_scaling
rope_count += 1
if rope_count != 1 or any(value.is_meta for value in list(model.parameters()) + list(model.buffers())):
raise ValueError("Unmaterialized parameters/buffers after exact RoPE recreation")
linear_count = 0
for name, module in model.backbone.named_modules():
if isinstance(module, torch.nn.Linear):
def forward(layer, inputs):
check_rss(rss_limit)
# No persistent cast/cache. Temporary buffers die after F.linear;
# the underlying allocator/mmap residency can still raise RSS.
return functional.linear(inputs.float(), layer.weight.float(), None if layer.bias is None else layer.bias.float())
module.forward = types.MethodType(forward, module)
linear_count += 1
else:
for parameter in module.parameters(recurse=False):
if parameter.dtype != torch.float32:
raise ValueError(f"Nonlinear source parameter is not FP32: {name}")
if not linear_count:
raise ValueError("No backbone linear projections were wrapped")
for layer in model.backbone.layers:
layer.register_forward_hook(lambda module, arguments, output: check_rss(rss_limit) and None)
model.eval()
tokenizer = AutoTokenizer.from_pretrained(source, local_files_only=True, trust_remote_code=False)
return model, tokenizer, encode, collate, linear_count
def verify_smaller_proofs(paths):
reports = [json.loads(Path(path).read_text()) for path in paths]
if {report.get("variant") for report in reports} != {"kai", "eos", "sol"} or len(reports) != 3:
raise ValueError("Nox requires one staged-vs-saved-FP32 proof for Kai, Eos and Sol")
for report in reports:
if report.get("method") != METHOD or report.get("verification", {}).get("status") != "passed" or report.get("source_revision") != PINS[report["variant"]][1]:
raise ValueError("Unverified/wrong smaller-model staged reference proof")
verification = report["verification"]
if not report.get("reports") or verification.get("failed_fixtures") or verification.get("max_logit_abs_error", math.inf) > 1e-5 or verification.get("max_probability_abs_error", math.inf) > 1e-6 or verification.get("max_hidden_abs_error", math.inf) > 1e-5 or not all(row.get("saved_upstream_fp32_comparison", {}).get("passed") is True for row in report["reports"]):
raise ValueError("Smaller-model proof lacks strict actual comparisons")
def run(args):
if not args.execute:
raise ValueError("Prototype performs no model work without explicit --execute")
if args.variant == "nox":
if not args.max_rss_bytes or args.verify_against:
raise ValueError("Nox requires a peak-RSS limit and cannot claim saved upstream FP32 parity")
verify_smaller_proofs(args.smaller_proof)
os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_TELEMETRY="1", HF_DEACTIVATE_ASYNC_LOAD="1")
source, hashes, backbone, decision, manifest = verify_source(args.source, args.variant)
inventory, shards = tensor_inventory(source)
fixtures = json.loads(args.fixtures.read_text())
if not isinstance(fixtures, list) or not fixtures or len({row["id"] for row in fixtures}) != len(fixtures):
raise ValueError("Nonempty unique-ID fixture list required")
saved = None
if args.verify_against:
saved = json.loads(args.verify_against.read_text())
if saved.get("variant") != args.variant or saved.get("source_revision") != PINS[args.variant][1] or not saved.get("method", "").startswith("upstream CPU FP32 causal SDPA"):
raise ValueError("Saved report is not the pinned actual CPU-FP32 reference")
baseline = {row["id"]: row for row in saved["reports"] if row["precision"] == "float32-baseline"}
if set(baseline) != {row["id"] for row in fixtures}:
raise ValueError("Saved FP32 fixture IDs differ; do not silently drop cases")
saved_readout = {row["id"]:row for row in saved.get("source_reference",[])}
if set(saved_readout) != set(baseline):
raise ValueError("Saved actual FP32 token/endpoint/hidden reference is required")
rss_before = peak_rss_bytes()
start = time.perf_counter()
model, tokenizer, encode, collate, linear_count = prepare_model(source, backbone, decision, inventory, shards, args.max_rss_bytes)
preparation_seconds = time.perf_counter() - start
import torch
observed = {}
hook = model.backbone.register_forward_hook(lambda module,arguments,output:observed.update(hidden=output.last_hidden_state.detach()))
results, failures = [], []
bias = json.loads((source / "score_bias.json").read_text()) if (source / "score_bias.json").exists() else None
for fixture in fixtures:
row = dict(fixture, label=0, family="synthetic-parity")
encoded = encode(row, tokenizer, PINS[args.variant][3])
batch = collate([encoded], tokenizer.pad_token_id or tokenizer.eos_token_id)
start = time.perf_counter()
with torch.inference_mode():
raw = model(**batch)[0, :len(row["options"])].float().tolist()
hidden=observed.pop("hidden")
vectors=torch.cat([hidden[0,encoded["candidate_positions"]],hidden[0,encoded["query_position"]][None]],dim=0).float().tolist()
del hidden
if not all(math.isfinite(value) for value in raw):
raise ValueError("Nonfinite staged candidate logits")
logits = list(raw)
if row["task_type"] == "score" and bias:
offset = bias.get("offsets", {}).get(str(len(logits)))
if offset:
logits = [value + offset[int(option["key"])] for value, option in zip(logits, row["options"])]
exponentials = [math.exp(value - max(logits)) for value in logits]
probabilities = [value / sum(exponentials) for value in exponentials]
record = {"id": row["id"], "ids": encoded["ids"], "candidate_positions": encoded["candidate_positions"], "query_position": encoded["query_position"], "token_ids_sha256": encoded["token_ids_sha256"], "prompt_sha256": encoded["prompt_sha256"], "keys": encoded["keys"], "raw_logits": raw, "score_bias_corrected_logits": logits, "probabilities": dict(zip(encoded["keys"], probabilities)), "cpu_seconds": time.perf_counter() - start, "whole_process_peak_rss_bytes": check_rss(args.max_rss_bytes)}
if saved is not None:
reference = baseline[row["id"]]
readout = saved_readout[row["id"]]
if any(readout[key] != encoded[key] for key in ("ids","candidate_positions","query_position","token_ids_sha256")):
raise ValueError("Saved exact token/endpoint/query positions differ")
if reference["token_ids_sha256"] != encoded["token_ids_sha256"] or reference["input_tokens"] != len(encoded["ids"]):
raise ValueError("Saved reference token sequence differs")
expected = reference["reference"]
if len(expected["logits"]) != len(logits) or set(expected["probabilities"]) != set(encoded["keys"]):
raise ValueError("Saved reference option menu differs")
if not all(math.isfinite(value) for value in expected["logits"]) or not all(math.isfinite(value) for value in expected["probabilities"].values()):
raise ValueError("Saved reference contains nonfinite scores")
logit_error = max(abs(a - b) for a, b in zip(logits, expected["logits"]))
probability_error = max(abs(p - expected["probabilities"][k]) for k, p in zip(encoded["keys"], probabilities))
passed = logit_error <= args.max_logit_error and probability_error <= args.max_probability_error
if len(vectors)!=len(readout["hidden_readout"]) or any(len(a)!=len(b) for a,b in zip(vectors,readout["hidden_readout"])):
raise ValueError("Saved endpoint/query hidden shapes differ")
hidden_deltas=[a-b for first,second in zip(vectors,readout["hidden_readout"]) for a,b in zip(first,second)]
hidden_max=max(abs(value) for value in hidden_deltas)
hidden_rms=math.sqrt(sum(value*value for value in hidden_deltas)/len(hidden_deltas))
passed = passed and hidden_max <= args.max_hidden_error
record["saved_upstream_fp32_comparison"] = {"max_logit_abs_error": logit_error, "max_probability_abs_error": probability_error,
"hidden_readout_max_abs_error":hidden_max,"hidden_readout_rms_error":hidden_rms,"passed": passed}
if not passed:
failures.append(row["id"])
results.append(record)
print("staged", row["id"], raw, flush=True)
report = {"variant": args.variant, "source_repo": PINS[args.variant][0], "source_revision": PINS[args.variant][1], "method": METHOD, "upstream_loading_modified": True, "validation_level": "experimental-staged-source-reference", "source_hashes": hashes, "tensor_inventory": inventory, "source_storage_bytes": sum(value["storage_bytes"] for value in inventory.values()), "source_manifest_model_sha256": manifest["identity"]["model_sha256"], "prototype_sha256": file_sha(__file__), "fixtures_sha256": file_sha(args.fixtures), "platform": platform.platform(), "architecture": platform.machine(), "environment": {name: importlib.metadata.version(name) for name in ("torch", "transformers", "safetensors", "tokenizers")}, "backbone_linear_wrappers": linear_count, "preparation_seconds": preparation_seconds, "whole_process_peak_rss_before_bytes": rss_before, "whole_process_peak_rss_bytes": peak_rss_bytes(), "memory_caveat": "Whole-process peak RSS includes resident source mappings, allocator and activations; checks do not prevent an individual allocation or prove safe RAM use.", "verification": {"status": "not-compared" if saved is None else "failed" if failures else "passed", "saved_report_sha256": file_sha(args.verify_against) if saved is not None else None, "max_logit_abs_error": args.max_logit_error, "max_probability_abs_error": args.max_probability_error, "max_hidden_abs_error": args.max_hidden_error, "failed_fixtures": failures}, "reports": results}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
hook.remove()
if failures:
raise ValueError(f"Staged reference differs from saved upstream FP32: {failures}; report retained")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("variant", choices=PINS)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--fixtures", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--verify-against", type=Path)
parser.add_argument("--smaller-proof", type=Path, nargs="*", default=[])
parser.add_argument("--max-rss-bytes", type=int, default=0)
parser.add_argument("--max-logit-error", type=float, default=1e-5)
parser.add_argument("--max-probability-error", type=float, default=1e-6)
parser.add_argument("--max-hidden-error", type=float, default=1e-5)
args = parser.parse_args()
if args.max_rss_bytes < 0 or not 0 < args.max_logit_error < math.inf or not 0 < args.max_probability_error < math.inf or not 0 < args.max_hidden_error < math.inf:
parser.error("Invalid memory/numerical limits")
run(args)
if __name__ == "__main__":
main()