"""Merged decision-model inference using Transformers and PyTorch.""" from __future__ import annotations import hashlib import json from pathlib import Path from typing import Any from protocol import encode class DecisionModel: """Load a self-contained checkpoint and return typed decision probabilities.""" def __init__(self, folder: str | Path | None = None) -> None: import torch from safetensors.torch import load_file from transformers import AutoModel, AutoTokenizer from transformers.masking_utils import create_recurrent_attention_mask self.folder = Path(folder or Path(__file__).parent).resolve() self.torch = torch self._verify_files() self.config = json.loads((self.folder / "decision_config.json").read_text()) self.tokenizer = AutoTokenizer.from_pretrained( self.folder, local_files_only=True ) self.backbone = AutoModel.from_pretrained( self.folder, dtype=torch.bfloat16, device_map={"": 0}, attn_implementation="sdpa", local_files_only=True, ) self.model = self.backbone.language_model self.readout = load_file( str(self.folder / "readout.safetensors"), device="cpu" )["weight"].to(device="cuda:0", dtype=torch.float32) if tuple(self.readout.shape) != (255, 2560): raise ValueError("Expected a [255, 2560] linear readout") def attention(module, args, kwargs): mask = kwargs["attention_mask"] embeddings = module.embed_tokens(kwargs["input_ids"]) kwargs["attention_mask"] = { "full_attention": mask[:, None, None, :].bool(), "linear_attention": create_recurrent_attention_mask( config=module.config, inputs_embeds=embeddings, attention_mask=mask, past_key_values=None, ), } return args, kwargs self._attention = self.model.register_forward_pre_hook( attention, with_kwargs=True ) self.model.eval() def _verify_files(self) -> None: manifest = json.loads((self.folder / "manifest.json").read_text()) for name, expected in manifest["files"].items(): path = (self.folder / name).resolve() if not path.is_relative_to(self.folder): raise ValueError(f"Invalid manifest path: {name}") with path.open("rb") as stream: actual = hashlib.file_digest(stream, "sha256").hexdigest() if actual != expected: raise ValueError(f"Checksum mismatch: {name}") def encode(self, state: Any, question_id: str, question: dict) -> dict: return encode( self.tokenizer, state, question_id, question, self.config["answer_codes"], ) def logits(self, items: list[dict]) -> list: """Score encoded questions without applying softmax.""" torch = self.torch from torch.nn.attention import SDPBackend, sdpa_kernel longest = max(len(item["input_ids"]) for item in items) ids = torch.full( (len(items), longest), self.tokenizer.pad_token_id, dtype=torch.long, device="cuda:0", ) mask = torch.zeros_like(ids) for index, item in enumerate(items): length = len(item["input_ids"]) ids[index, :length] = torch.tensor( item["input_ids"], device="cuda:0" ) mask[index, :length] = 1 backends = [ SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH, ] with torch.no_grad(), sdpa_kernel(backends), torch.autocast( "cuda", dtype=torch.bfloat16 ): hidden = self.model( input_ids=ids, attention_mask=mask, use_cache=False, return_dict=True, ).last_hidden_state return [ torch.nn.functional.linear( hidden[index, item["position"]].float(), self.readout )[: len(item["option_ids"])].float() for index, item in enumerate(items) ] def decide(self, request: dict) -> dict: """Evaluate one request; each question sees only its own schema.""" return self.decide_many([request])[0] def decide_many(self, requests: list[dict]) -> list[dict]: """Batch independent requests without changing their question schemas.""" answers = [{} for _ in requests] tokens = [0 for _ in requests] sequences = [0 for _ in requests] children = [] for owner, request in enumerate(requests): questions = request.get("questions") if "state" not in request or not isinstance(questions, dict) or not questions: raise ValueError("state and nonempty questions are required") if request.get("images") or request.get("videos"): raise ValueError("This release accepts text only") for key, question in questions.items(): item = self.encode(request["state"], key, question) children.append((owner, key, question, item)) tokens[owner] += len(item["input_ids"]) sequences[owner] += 1 offset = 0 while offset < len(children): batch = [] longest = 0 while offset < len(children): entry = children[offset] length = len(entry[3]["input_ids"]) if batch and ( len(batch) >= 4 or max(longest, length) * (len(batch) + 1) > 16384 ): break batch.append(entry) longest = max(longest, length) offset += 1 logits = self.logits([entry[3] for entry in batch]) for (owner, key, question, item), scores in zip(batch, logits): if not bool(self.torch.isfinite(scores).all()): raise ValueError("Nonfinite decision logits") probabilities = dict( zip(item["option_ids"], scores.softmax(-1).cpu().tolist()) ) if question["type"] == "noul": answer = {"type": "noul", "noul": probabilities["true"]} else: ordered = {key: probabilities[key] for key in question["criteria"]} answer = { "type": "choice", "choice": max(ordered, key=ordered.get), "probabilities": ordered, } answers[owner][key] = answer return [ { "model": "decision-4b-preview", "answers": answer, "usage": { "input_tokens": tokens[index], "output_tokens": 0, "sequences": sequences[index], }, } for index, answer in enumerate(answers) ]