EpochEcho's picture
Upload folder using huggingface_hub
883a747 verified
Raw History Blame Contribute Delete
7.37 kB
"""Merged decision-model inference using Transformers and PyTorch."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
from protocol import encode
class DecisionModel:
"""Load a self-contained checkpoint and return typed decision probabilities."""
def __init__(self, folder: str | Path | None = None) -> None:
import torch
from safetensors.torch import load_file
from transformers import AutoModel, AutoTokenizer
from transformers.masking_utils import create_recurrent_attention_mask
self.folder = Path(folder or Path(__file__).parent).resolve()
self.torch = torch
self._verify_files()
self.config = json.loads((self.folder / "decision_config.json").read_text())
self.tokenizer = AutoTokenizer.from_pretrained(
self.folder, local_files_only=True
)
self.backbone = AutoModel.from_pretrained(
self.folder,
dtype=torch.bfloat16,
device_map={"": 0},
attn_implementation="sdpa",
local_files_only=True,
)
self.model = self.backbone.language_model
self.readout = load_file(
str(self.folder / "readout.safetensors"), device="cpu"
)["weight"].to(device="cuda:0", dtype=torch.float32)
if tuple(self.readout.shape) != (255, 2560):
raise ValueError("Expected a [255, 2560] linear readout")
def attention(module, args, kwargs):
mask = kwargs["attention_mask"]
embeddings = module.embed_tokens(kwargs["input_ids"])
kwargs["attention_mask"] = {
"full_attention": mask[:, None, None, :].bool(),
"linear_attention": create_recurrent_attention_mask(
config=module.config,
inputs_embeds=embeddings,
attention_mask=mask,
past_key_values=None,
),
}
return args, kwargs
self._attention = self.model.register_forward_pre_hook(
attention, with_kwargs=True
)
self.model.eval()
def _verify_files(self) -> None:
manifest = json.loads((self.folder / "manifest.json").read_text())
for name, expected in manifest["files"].items():
path = (self.folder / name).resolve()
if not path.is_relative_to(self.folder):
raise ValueError(f"Invalid manifest path: {name}")
with path.open("rb") as stream:
actual = hashlib.file_digest(stream, "sha256").hexdigest()
if actual != expected:
raise ValueError(f"Checksum mismatch: {name}")
def encode(self, state: Any, question_id: str, question: dict) -> dict:
return encode(
self.tokenizer,
state,
question_id,
question,
self.config["answer_codes"],
)
def logits(self, items: list[dict]) -> list:
"""Score encoded questions without applying softmax."""
torch = self.torch
from torch.nn.attention import SDPBackend, sdpa_kernel
longest = max(len(item["input_ids"]) for item in items)
ids = torch.full(
(len(items), longest),
self.tokenizer.pad_token_id,
dtype=torch.long,
device="cuda:0",
)
mask = torch.zeros_like(ids)
for index, item in enumerate(items):
length = len(item["input_ids"])
ids[index, :length] = torch.tensor(
item["input_ids"], device="cuda:0"
)
mask[index, :length] = 1
backends = [
SDPBackend.FLASH_ATTENTION,
SDPBackend.EFFICIENT_ATTENTION,
SDPBackend.MATH,
]
with torch.no_grad(), sdpa_kernel(backends), torch.autocast(
"cuda", dtype=torch.bfloat16
):
hidden = self.model(
input_ids=ids,
attention_mask=mask,
use_cache=False,
return_dict=True,
).last_hidden_state
return [
torch.nn.functional.linear(
hidden[index, item["position"]].float(), self.readout
)[: len(item["option_ids"])].float()
for index, item in enumerate(items)
]
def decide(self, request: dict) -> dict:
"""Evaluate one request; each question sees only its own schema."""
return self.decide_many([request])[0]
def decide_many(self, requests: list[dict]) -> list[dict]:
"""Batch independent requests without changing their question schemas."""
answers = [{} for _ in requests]
tokens = [0 for _ in requests]
sequences = [0 for _ in requests]
children = []
for owner, request in enumerate(requests):
questions = request.get("questions")
if "state" not in request or not isinstance(questions, dict) or not questions:
raise ValueError("state and nonempty questions are required")
if request.get("images") or request.get("videos"):
raise ValueError("This release accepts text only")
for key, question in questions.items():
item = self.encode(request["state"], key, question)
children.append((owner, key, question, item))
tokens[owner] += len(item["input_ids"])
sequences[owner] += 1
offset = 0
while offset < len(children):
batch = []
longest = 0
while offset < len(children):
entry = children[offset]
length = len(entry[3]["input_ids"])
if batch and (
len(batch) >= 4
or max(longest, length) * (len(batch) + 1) > 16384
):
break
batch.append(entry)
longest = max(longest, length)
offset += 1
logits = self.logits([entry[3] for entry in batch])
for (owner, key, question, item), scores in zip(batch, logits):
if not bool(self.torch.isfinite(scores).all()):
raise ValueError("Nonfinite decision logits")
probabilities = dict(
zip(item["option_ids"], scores.softmax(-1).cpu().tolist())
)
if question["type"] == "noul":
answer = {"type": "noul", "noul": probabilities["true"]}
else:
ordered = {key: probabilities[key] for key in question["criteria"]}
answer = {
"type": "choice",
"choice": max(ordered, key=ordered.get),
"probabilities": ordered,
}
answers[owner][key] = answer
return [
{
"model": "decision-4b-preview",
"answers": answer,
"usage": {
"input_tokens": tokens[index],
"output_tokens": 0,
"sequences": sequences[index],
},
}
for index, answer in enumerate(answers)
]