Bazza / systemone.py
AdrianJohnston's picture adrianjohnstonaus's picture
Release Bazza 0.5.0 decision head and standalone inference code (#1)
0f99db6
Raw History Blame Contribute Delete
12.4 kB
"""System 1 inference using the state/questions/criteria request format."""
import json
import math
from collections.abc import Mapping
from pathlib import Path
import torch
from data import encode_input
from model import Bazza, BazzaDecisionHead, Gemma4
from model.gemma_backbone import matches_gemma_source
from utils import autocast, file_hash
def _checkpoint_settings(saved):
"""Validate the checkpoint identity and required architecture settings."""
from model import gemma
config = saved.get("config", {})
state = saved.get("head")
if not isinstance(state, dict) or "letter.weight" not in state:
raise ValueError("Expected a Bazza decision-head checkpoint")
for name, expected in [
("backbone_checkpoint", gemma.CHECKPOINT),
("backbone_revision", gemma.REVISION),
]:
if config.get(name) != expected:
raise ValueError("Checkpoint backbone identity differs: " + name)
required = [
"hidden_width",
"attention_heads",
"low_steps",
"high_cycles",
"gate_width",
"adapter_rank",
"quantization",
]
missing = [name for name in required if name not in config]
if missing:
raise ValueError(
"Checkpoint lacks architecture settings: " + ", ".join(missing)
)
if config["quantization"] == "bnb-nf4":
from importlib.util import find_spec
if find_spec("bitsandbytes") is None:
raise RuntimeError(
"NF4 inference requires bitsandbytes; install requirements.txt or build the Bazza Docker image"
)
return config, state
def _load_tokenizer(saved, weights):
"""Verify backbone source and tokenizer identity before loading the tokenizer."""
from tokenizers import Tokenizer
tokenizer_path = weights / "tokenizer.json"
expected_tokenizer = (
saved.get("identity", {}).get("readout_files", {}).get("tokenizer.json")
)
if (
expected_tokenizer is not None
and file_hash(tokenizer_path) != expected_tokenizer
):
raise ValueError("Tokenizer differs from the training checkpoint")
expected_source = (
saved.get("identity", {}).get("source_sha256", {}).get("model/gemma.py")
)
if expected_source is not None and not matches_gemma_source(expected_source):
raise ValueError("Bundled backbone differs from the training checkpoint")
return Tokenizer.from_file(str(tokenizer_path))
def _load_head(saved, config, state):
"""Construct the decision head and restore its trained parameters."""
softcap = saved.get("readout", {}).get("softcap", config.get("softcap"))
if softcap is None:
raise ValueError("Checkpoint lacks the readout softcap")
letter = state["letter.weight"]
if (
letter.ndim != 2
or letter.shape[0] != 26
or config.get("encoder_width") != letter.shape[1]
):
raise ValueError("Checkpoint encoder/readout dimensions differ")
head = BazzaDecisionHead(
letter,
softcap,
encoder_width=letter.shape[1],
width=config["hidden_width"],
attention_heads=config["attention_heads"],
low_steps=config["low_steps"],
high_cycles=config["high_cycles"],
gate_width=config["gate_width"],
rank=config["adapter_rank"],
)
head.load_state_dict(state, strict=True)
return head
class SystemOne:
"""Wrap a loaded Bazza model and tokenizer with a typed decision API."""
def __init__(self, model, tokenizer, *, max_length=4096):
if max_length < 1:
raise ValueError("max_length must be positive")
self.model = model.eval()
self.tokenizer = tokenizer
self.max_length = max_length
self.device = self.model.head.letter.weight.device
if self.device.type == "cpu" and self.model.head_precision == "bf16":
self.model.head.float()
self.model.head_precision = "fp32"
@classmethod
def from_checkpoint(
cls, checkpoint, *, backbone_path, device="cuda", max_length=4096
):
"""Load a trained decision checkpoint and the bundled live Gemma backbone.
Training data, feature caches and experiment configuration files are not
required. Architecture settings are read from the checkpoint itself.
"""
saved = torch.load(Path(checkpoint), map_location="cpu", weights_only=True)
return cls._from_saved(
saved, backbone_path=backbone_path, device=device, max_length=max_length
)
@classmethod
def from_pretrained(
cls,
model_id,
*,
revision=None,
backbone_path=None,
cache_dir=None,
token=None,
local_files_only=False,
device="cuda",
max_length=4096,
):
"""Load a head release and its pinned backbone from the Hugging Face Hub.
A local release directory and optional local backbone path also work.
Downloaded Python code is never executed; use this installed implementation.
"""
from huggingface_hub import snapshot_download
from safetensors.torch import load_file
download_options = dict(
cache_dir=cache_dir, token=token, local_files_only=local_files_only
)
directory = Path(model_id).expanduser()
if not directory.is_dir():
directory = Path(
snapshot_download(
repo_id=str(model_id),
revision=revision,
allow_patterns=["config.json", "model.safetensors"],
**download_options,
)
)
saved = json.loads((directory / "config.json").read_text())
if saved.get("model_type") != "bazza" or saved.get("format_version") != 1:
raise ValueError("Unsupported Bazza head release format")
weight_path = directory / "model.safetensors"
expected_hash = saved.get("weights", {}).get("sha256")
if (
not isinstance(expected_hash, str)
or file_hash(weight_path) != expected_hash
):
raise ValueError("Bazza head weight SHA-256 mismatch")
saved["head"] = load_file(str(weight_path), device="cpu")
config, _ = _checkpoint_settings(saved)
if backbone_path is None:
backbone_path = snapshot_download(
repo_id=config["backbone_checkpoint"],
revision=config["backbone_revision"],
allow_patterns=["config.json", "tokenizer.json", "model.safetensors"],
**download_options,
)
return cls._from_saved(
saved, backbone_path=backbone_path, device=device, max_length=max_length
)
@classmethod
def _from_saved(cls, saved, *, backbone_path, device, max_length):
"""Construct inference modules from validated checkpoint metadata."""
from model import gemma
config, state = _checkpoint_settings(saved)
weights = Path(backbone_path).expanduser().resolve()
tokenizer = _load_tokenizer(saved, weights)
identity = dict(
checkpoint=gemma.CHECKPOINT,
revision=gemma.REVISION,
weight_sha256=gemma.WEIGHT_SHA256,
quantization=config["quantization"],
implementation_sha256=file_hash(Path(gemma.__file__)),
)
backbone = Gemma4(
weights,
quantization=config["quantization"],
selection={"identity": identity},
)
head = _load_head(saved, config, state)
precision = (
config.get("precision", "bf16")
if torch.device(device).type == "cuda"
else "fp32"
)
if precision == "fp32":
head.float()
model = Bazza(backbone, head, head_precision=precision).to(device).eval()
backbone._load()
return cls(model, tokenizer, max_length=max_length)
def _prepare(self, state, questions):
# Validate the complete request before any model execution.
try:
json.dumps(state, allow_nan=False, sort_keys=True)
except (TypeError, ValueError) as error:
raise ValueError("state must be finite JSON-compatible data") from error
if not isinstance(questions, Mapping) or not questions:
raise ValueError("questions must be a nonempty mapping")
return [
self._prepare_question(state, name, question)
for name, question in questions.items()
]
def _prepare_question(self, state, name, question):
"""Validate one typed question and encode its candidate decisions."""
if not isinstance(name, str) or not name or not isinstance(question, Mapping):
raise ValueError("Each question requires a nonempty name and object")
kind = question.get("type")
if kind not in ("choice", "noul", "score"):
raise ValueError("Unsupported question type: " + str(kind))
instructions, criteria = (
question.get("instructions"),
question.get("criteria"),
)
if not isinstance(instructions, str) or not instructions.strip():
raise ValueError("Question instructions must be nonempty text")
if not isinstance(criteria, Mapping) or not 2 <= len(criteria) <= 26:
raise ValueError("criteria must contain between 2 and 26 candidates")
if any(
not isinstance(k, str) or not k or not isinstance(v, str) or not v.strip()
for k, v in criteria.items()
):
raise ValueError("Criterion keys and descriptions must be nonempty strings")
keys = list(criteria)
if kind == "noul" and set(keys) != {"true", "false"}:
raise ValueError("noul criteria must contain true and false")
if kind == "score":
try:
levels = [float(k) for k in keys]
except ValueError as error:
raise ValueError("score criteria require numeric keys") from error
if not all(math.isfinite(v) for v in levels) or len(set(levels)) != len(
levels
):
raise ValueError("score criteria require distinct finite numeric keys")
row = dict(
id=name,
group_id=name,
source="systemone",
kind=kind,
state=state,
question=instructions,
options=list(criteria.values()),
)
encoded = encode_input(row, self.tokenizer, self.max_length)
if encoded is None:
raise ValueError(
f"Question {name!r} exceeds max_length={self.max_length}; input was not truncated"
)
return name, kind, keys, encoded
@torch.inference_mode()
def decide(self, *, state, questions):
"""Return answers keyed by question name; candidate order is preserved.
Each question is scored as a separate request, matching training. Values
in probabilities are the model's softmax scores, without calibration.
"""
prepared = self._prepare(state, questions)
answers = {}
for name, kind, keys, row in prepared:
batch = self.model.backbone.prepare_batch([row], "inference").to(
self.device
)
with autocast(self.device, self.model.head_precision):
logits = self.model(batch).logits[0, : len(keys)].float()
if not torch.isfinite(logits).all():
raise FloatingPointError("Non-finite decision logits")
probabilities = dict(zip(keys, logits.softmax(-1).cpu().tolist()))
chosen = keys[int(logits.argmax())]
answer = dict(
type=kind, choice=chosen, value=chosen, probabilities=probabilities
)
if kind == "noul":
answer["noul"] = probabilities["true"]
answer["value"] = probabilities["true"] > 0.5
elif kind == "score":
answer["score"] = sum(float(k) * v for k, v in probabilities.items())
answer["value"] = answer["score"]
answers[name] = answer
return {"answers": answers}
__call__ = decide