Bazza / model /model.py
AdrianJohnston's picture adrianjohnstonaus's picture
Release Bazza 0.5.0 decision head and standalone inference code (#1)
0f99db6
Raw History Blame Contribute Delete
2.75 kB
"""Bazza composition; state-dict keys of the existing mixed head are unchanged."""
from dataclasses import dataclass
import torch
from torch import nn
from .readers import make_head
from .readout import LetterReadout, MixedReadout
class BazzaDecisionHead(MixedReadout):
def __init__(
self,
letter_weight,
softcap,
encoder_width=3840,
width=1024,
attention_heads=4,
low_steps=2,
high_cycles=2,
gate_width=32,
rank=64,
):
decision = make_head(
encoder_width, width, attention_heads, low_steps, high_cycles, "bigru"
)
super().__init__(
decision,
LetterReadout(letter_weight, softcap),
encoder_width,
width,
gate_width,
rank,
)
def forward(self, hidden, attention_mask, option_positions, option_mask):
return super().forward(hidden, attention_mask, option_positions, option_mask)
@dataclass
class DecisionOutput:
logits: torch.Tensor
option_mask: torch.Tensor
class Bazza(nn.Module):
"""One frozen backbone plus a trainable head; the original LM head is retained.
A backbone adapter must return normalized final token states. Generation and
System 1/System 2 routing are intentionally separate future operations.
"""
def __init__(self, backbone, head, *, head_precision="bf16"):
super().__init__()
self.backbone = backbone.requires_grad_(False).eval()
self.head = head
self.head_precision = head_precision
def train(self, mode=True):
super().train(mode)
self.backbone.eval()
return self
@property
def checkpoint_module(self):
"""Reference the frozen backbone by identity; save only the trained head."""
return self.head
def forward(self, batch):
batch = batch.to(self.head.letter.weight.device)
# Match the exporter/live path: backbone and original RMSNorm run
# outside the head's autocast context.
with (
torch.no_grad(),
torch.autocast(batch.attention_mask.device.type, enabled=False),
):
features = self.backbone(batch)
if not torch.isfinite(features.hidden).all():
raise FloatingPointError("Non-finite frozen features")
hidden = (
features.hidden.float()
if self.head_precision == "fp32"
else features.hidden
)
logits = self.head(
hidden, batch.attention_mask, batch.option_positions, batch.option_mask
)
return DecisionOutput(logits, batch.option_mask)
def diagnostics(self):
return self.head.diagnostics()