Bazza / model /reasoner.py
AdrianJohnston's picture adrianjohnstonaus's picture
Release Bazza 0.5.0 decision head and standalone inference code (#1)
0f99db6
Raw History Blame Contribute Delete
15.6 kB
"""Two-timescale action/slot reasoning with a small copy-capable value decoder.
This is an HRM-inspired candidate, not a reproduction of the puzzle model's
training algorithm. The encoder token states remain available to the decoder.
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
from torch import nn
from torch.nn import functional as F
@dataclass(frozen=True)
class ReasoningConfig:
encoder_width: int = 1024
width: int = 256
attention_heads: int = 4
low_steps: int = 2
high_cycles: int = 2
slot_gate_classes: int = 4 # omit, select, copy, generate
def __post_init__(self) -> None:
if self.width < 1 or self.width % self.attention_heads:
raise ValueError("reasoning width must be divisible by attention_heads")
if self.encoder_width < 1 or self.low_steps < 1 or self.high_cycles < 1:
raise ValueError("encoder width and recurrence counts must be positive")
@dataclass
class ReasoningOutput:
action_logits: torch.Tensor
slot_gate_logits: torch.Tensor
slot_states: torch.Tensor
memory: torch.Tensor
high_state: torch.Tensor
class LowLevelUpdate(nn.Module):
"""Shared update applied repeatedly to the small action/slot workspace."""
def __init__(self, width: int, attention_heads: int) -> None:
super().__init__()
self.high_condition = nn.Linear(width, width)
self.self_norm = nn.LayerNorm(width)
self.self_attention = nn.MultiheadAttention(
width, attention_heads, batch_first=True
)
self.cross_norm = nn.LayerNorm(width)
self.cross_attention = nn.MultiheadAttention(
width, attention_heads, batch_first=True
)
self.ffn_norm = nn.LayerNorm(width)
self.ffn = nn.Sequential(
nn.Linear(width, 2 * width), nn.GELU(), nn.Linear(2 * width, width)
)
def forward(
self,
low: torch.Tensor,
high: torch.Tensor,
memory: torch.Tensor,
workspace_mask: torch.Tensor,
memory_mask: torch.Tensor,
) -> torch.Tensor:
conditioned = low + self.high_condition(high)[:, None, :]
query = self.self_norm(conditioned)
update, _ = self.self_attention(
query,
query,
query,
key_padding_mask=~workspace_mask,
need_weights=False,
)
low = low + update
update, _ = self.cross_attention(
self.cross_norm(low),
memory,
memory,
key_padding_mask=~memory_mask,
need_weights=False,
)
low = low + update
low = low + self.ffn(self.ffn_norm(low))
return low.masked_fill(~workspace_mask[:, :, None], 0)
class ReasoningHead(nn.Module):
"""Slow global state and fast per-action/per-slot states over encoder memory."""
def __init__(self, config: ReasoningConfig) -> None:
super().__init__()
self.config = config
self.project = nn.Linear(config.encoder_width, config.width)
self.high_init = nn.Linear(config.width, config.width)
self.low_update = LowLevelUpdate(config.width, config.attention_heads)
self.high_update = nn.GRUCell(config.width, config.width)
self.action_norm = nn.LayerNorm(config.width)
self.action_score = nn.Linear(config.width, 1)
self.slot_norm = nn.LayerNorm(config.width)
self.slot_gate = nn.Linear(config.width, config.slot_gate_classes)
@staticmethod
def _gather(memory: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
if positions.ndim != 2 or positions.shape[0] != memory.shape[0]:
raise ValueError("marker positions must have shape [batch, markers]")
if torch.any(positions < 0) or torch.any(positions >= memory.shape[1]):
raise ValueError("marker positions are outside encoder memory")
index = positions[:, :, None].expand(-1, -1, memory.shape[-1])
return memory.gather(dim=1, index=index)
def forward(
self,
encoder_hidden: torch.Tensor,
attention_mask: torch.Tensor,
action_positions: torch.Tensor,
slot_positions: torch.Tensor,
action_mask: torch.Tensor | None = None,
slot_mask: torch.Tensor | None = None,
) -> ReasoningOutput:
if (
encoder_hidden.ndim != 3
or encoder_hidden.shape[-1] != self.config.encoder_width
):
raise ValueError("encoder_hidden has the wrong shape")
if attention_mask.shape != encoder_hidden.shape[:2]:
raise ValueError("attention_mask has the wrong shape")
if action_positions.shape[1] < 1:
raise ValueError("at least one action is required")
if action_mask is None:
action_mask = torch.ones_like(action_positions, dtype=torch.bool)
if slot_mask is None:
slot_mask = torch.ones_like(slot_positions, dtype=torch.bool)
if (
action_mask.shape != action_positions.shape
or slot_mask.shape != slot_positions.shape
):
raise ValueError("marker masks must match marker positions")
memory_mask = attention_mask.bool()
if not torch.all(memory_mask.any(dim=1)):
raise ValueError("each input needs at least one unmasked token")
workspace_mask = torch.cat((action_mask.bool(), slot_mask.bool()), dim=1)
if not torch.all(workspace_mask.any(dim=1)):
raise ValueError("each input needs an unmasked marker")
memory = self.project(encoder_hidden)
low = torch.cat(
(
self._gather(memory, action_positions),
self._gather(memory, slot_positions),
),
dim=1,
)
low = low.masked_fill(~workspace_mask[:, :, None], 0)
high = self.high_init(memory[:, 0])
for _ in range(self.config.high_cycles):
for _ in range(self.config.low_steps):
low = self.low_update(low, high, memory, workspace_mask, memory_mask)
pooled = (low * workspace_mask[:, :, None]).sum(dim=1) / workspace_mask.sum(
dim=1,
keepdim=True,
).clamp_min(1)
high = self.high_update(pooled, high)
n_actions = action_positions.shape[1]
action_states = low[:, :n_actions] + high[:, None, :]
slot_states = low[:, n_actions:] + high[:, None, :]
action_logits = (
self.action_score(self.action_norm(action_states)).squeeze(-1).float()
)
action_logits = action_logits.masked_fill(~action_mask.bool(), -1e4)
slot_gate_logits = self.slot_gate(self.slot_norm(slot_states)).float()
slot_gate_logits = slot_gate_logits.masked_fill(
~slot_mask.bool()[:, :, None], -1e4
)
return ReasoningOutput(
action_logits, slot_gate_logits, slot_states, memory, high
)
class SmallValueDecoder(nn.Module):
"""Short token decoder with a learned choice between generating and copying."""
def __init__(self, vocab_size: int, width: int = 256) -> None:
super().__init__()
if vocab_size < 2 or width < 1:
raise ValueError("vocab_size and width must be positive")
self.vocab_size = vocab_size
self.width = width
self.embedding = nn.Embedding(vocab_size, width)
self.memory_key = nn.Linear(width, width, bias=False)
self.query = nn.Linear(width, width, bias=False)
self.initial = nn.Linear(width, width)
self.cell = nn.GRUCell(2 * width, width)
self.output_norm = nn.LayerNorm(width)
self.output_bias = nn.Parameter(torch.zeros(vocab_size))
self.generate_gate = nn.Linear(2 * width, 1)
def _prepare(
self,
slot_states: torch.Tensor,
memory: torch.Tensor,
source_ids: torch.Tensor,
source_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if slot_states.ndim != 3 or memory.ndim != 3:
raise ValueError(
"slot_states and memory must have [batch, length, width] shape"
)
batch, slots, width = slot_states.shape
if width != self.width or memory.shape[0] != batch or memory.shape[2] != width:
raise ValueError("decoder hidden widths or batch sizes do not match")
if (
source_ids.shape != memory.shape[:2]
or source_mask.shape != source_ids.shape
):
raise ValueError("source IDs and mask must match memory")
if slots < 1 or not torch.all(source_mask.bool().any(dim=1)):
raise ValueError("at least one slot and one source token are required")
if torch.any(source_ids < 0) or torch.any(source_ids >= self.vocab_size):
raise ValueError("source token ID is outside decoder vocabulary")
expanded_memory = (
memory[:, None]
.expand(-1, slots, -1, -1)
.reshape(
batch * slots,
memory.shape[1],
width,
)
)
expanded_ids = (
source_ids[:, None].expand(-1, slots, -1).reshape(batch * slots, -1)
)
expanded_mask = (
source_mask.bool()[:, None].expand(-1, slots, -1).reshape(batch * slots, -1)
)
hidden = torch.tanh(self.initial(slot_states.reshape(batch * slots, width)))
keys = self.memory_key(expanded_memory)
return hidden, expanded_memory, keys, expanded_ids, expanded_mask
def _step(
self,
token: torch.Tensor,
hidden: torch.Tensor,
memory: torch.Tensor,
keys: torch.Tensor,
source_ids: torch.Tensor,
source_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
scores = torch.bmm(keys, self.query(hidden)[:, :, None]).squeeze(-1)
scores = scores.float().div(self.width**0.5).masked_fill(~source_mask, -1e4)
attention = torch.softmax(scores, dim=-1)
context = torch.bmm(attention.to(memory.dtype)[:, None, :], memory).squeeze(1)
token_embedding = self.embedding(token).to(context.dtype)
hidden = self.cell(torch.cat((token_embedding, context), dim=-1), hidden)
logits = F.linear(
self.output_norm(hidden), self.embedding.weight, self.output_bias
)
generated = torch.softmax(logits.float(), dim=-1)
copied = torch.zeros_like(generated).scatter_add_(1, source_ids, attention)
gate = torch.sigmoid(
self.generate_gate(torch.cat((hidden, context), dim=-1))
).float()
probabilities = gate * generated + (1 - gate) * copied
return probabilities, hidden
def generate(
self,
slot_states: torch.Tensor,
memory: torch.Tensor,
source_ids: torch.Tensor,
source_mask: torch.Tensor,
*,
bos_token_id: int,
max_new_tokens: int,
eos_token_id: int | None = None,
) -> torch.Tensor:
"""Greedy fixed-length decoding; fixed length makes latency comparable."""
if not 0 <= bos_token_id < self.vocab_size or max_new_tokens < 1:
raise ValueError("invalid BOS token or decoding length")
batch, slots, _ = slot_states.shape
hidden, expanded_memory, keys, expanded_ids, expanded_mask = self._prepare(
slot_states,
memory,
source_ids,
source_mask,
)
token = torch.full(
(batch * slots,), bos_token_id, device=source_ids.device, dtype=torch.long
)
result = []
finished = torch.zeros(
batch * slots, device=source_ids.device, dtype=torch.bool
)
for _ in range(max_new_tokens):
probabilities, hidden = self._step(
token,
hidden,
expanded_memory,
keys,
expanded_ids,
expanded_mask,
)
token = probabilities.argmax(dim=-1)
result.append(token)
if eos_token_id is not None:
finished |= token == eos_token_id
if torch.all(finished):
break
return torch.stack(result, dim=-1).reshape(batch, slots, len(result))
def teacher_forced_nll(
self,
slot_states: torch.Tensor,
memory: torch.Tensor,
source_ids: torch.Tensor,
source_mask: torch.Tensor,
target_ids: torch.Tensor,
*,
bos_token_id: int,
pad_token_id: int | None = None,
) -> torch.Tensor:
"""Mean token NLL for later supervised training; no training is run here."""
batch, slots, steps = target_ids.shape
if slot_states.shape[:2] != (batch, slots) or steps < 1:
raise ValueError("target IDs do not match slot states")
hidden, expanded_memory, keys, expanded_ids, expanded_mask = self._prepare(
slot_states,
memory,
source_ids,
source_mask,
)
target = target_ids.reshape(batch * slots, steps)
token = torch.full(
(batch * slots,), bos_token_id, device=target.device, dtype=torch.long
)
losses = []
for step in range(steps):
probabilities, hidden = self._step(
token,
hidden,
expanded_memory,
keys,
expanded_ids,
expanded_mask,
)
gold_probability = probabilities.gather(
1, target[:, step : step + 1]
).squeeze(1)
losses.append(-gold_probability.clamp_min(1e-12).log())
token = target[:, step]
losses = torch.stack(losses, dim=-1)
if pad_token_id is None:
return losses.mean()
valid = target != pad_token_id
return (losses * valid).sum() / valid.sum().clamp_min(1)
class StructuredAgent(nn.Module):
"""Encoder, recurrent workspace, typed heads, and optional value decoder."""
def __init__(
self, encoder: nn.Module, config: ReasoningConfig, vocab_size: int
) -> None:
super().__init__()
self.encoder = encoder
self.reasoner = ReasoningHead(config)
self.decoder = SmallValueDecoder(vocab_size, config.width)
def forward(
self,
input_ids: torch.Tensor,
attention_mask: torch.Tensor,
action_positions: torch.Tensor,
slot_positions: torch.Tensor,
*,
generated_slot_indices: torch.Tensor | None = None,
bos_token_id: int = 0,
max_new_tokens: int = 16,
) -> tuple[ReasoningOutput, torch.Tensor | None]:
hidden = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
result = self.reasoner(hidden, attention_mask, action_positions, slot_positions)
if generated_slot_indices is None:
return result, None
if generated_slot_indices.ndim != 1 or generated_slot_indices.numel() < 1:
raise ValueError("generated_slot_indices must be a nonempty vector")
slots = result.slot_states.index_select(1, generated_slot_indices)
values = self.decoder.generate(
slots,
result.memory,
input_ids,
attention_mask,
bos_token_id=bos_token_id,
max_new_tokens=max_new_tokens,
)
return result, values