Bazza / model /readout.py
AdrianJohnston's picture adrianjohnstonaus's picture
Release Bazza 0.5.0 decision head and standalone inference code (#1)
0f99db6
Raw History Blame Contribute Delete
8.73 kB
"""Frozen letter prior, request gate and residual answer projection."""
import math
import torch
from torch import nn
from torch.nn import functional as F
class LetterReadout(nn.Module):
def __init__(self, weight, softcap):
super().__init__()
self.register_buffer("weight", weight.clone())
self.softcap = softcap
def forward(self, hidden, mask, positions, option_mask):
last = mask.long().sum(-1) - 1
final = hidden[torch.arange(len(hidden), device=hidden.device), last]
logits = F.linear(final, self.weight[: option_mask.shape[1]]).float()
if self.softcap:
logits = self.softcap * (logits / self.softcap).tanh()
return logits.masked_fill(~option_mask, -1e4)
class HybridReadout(nn.Module):
def __init__(self, decision, letter, initial_prior=0.1):
super().__init__()
self.decision, self.letter = decision, letter
# Deterministic scalar: adding this branch must not change initialization
# of the shared decision core. AdamW uses the same auxiliary LR as before.
self.prior_gate = nn.Parameter(torch.tensor(math.atanh(initial_prior)))
def forward(self, hidden, mask, positions, option_mask):
decision = self.decision(hidden, mask, positions, option_mask)
prior = self.letter(hidden, mask, positions, option_mask)
return (decision + self.prior_gate.tanh() * prior).masked_fill(
~option_mask, -1e4
)
def final_state(hidden, mask):
last = mask.long().sum(-1) - 1
return hidden[torch.arange(len(hidden), device=hidden.device), last]
def confidence_features(logits, mask):
# Confidence is observable at inference. Detach to stop the scorer learning
# to manipulate the gate indirectly through confidence, rather than answers.
probs = logits.detach().float().masked_fill(~mask, -1e4).softmax(-1)
count = mask.sum(-1).float()
entropy = -(probs * probs.clamp_min(1e-8).log()).sum(-1) / count.log()
top = probs.topk(2, dim=-1).values
return torch.stack((entropy, top[:, 0] - top[:, 1]), -1)
class RequestGateReadout(nn.Module):
def __init__(self, decision, letter, encoder_width=3840, gate_width=32):
super().__init__()
self.decision, self.letter = decision, letter
self.request_norm = nn.LayerNorm(encoder_width)
self.request_project = nn.Linear(encoder_width, gate_width)
# Name the gate's output projection action_score so the existing optimizer
# puts it on the unchanged auxiliary AdamW LR, as for the original scorer.
self.action_score = nn.Linear(gate_width + 5, 1)
nn.init.zeros_(self.action_score.weight)
nn.init.constant_(self.action_score.bias, math.log(0.1 / 0.9))
self.last_gate = None
def forward(self, hidden, mask, positions, option_mask):
decision = self.decision(hidden, mask, positions, option_mask)
prior = self.letter(hidden, mask, positions, option_mask)
state = F.silu(
self.request_project(self.request_norm(final_state(hidden, mask)))
)
stats = torch.cat(
(
confidence_features(decision, option_mask),
confidence_features(prior, option_mask),
option_mask.sum(-1, keepdim=True).float().log(),
),
-1,
)
gate = self.action_score(torch.cat((state, stats), -1)).float().sigmoid()
self.last_gate = gate.detach()
return (decision + gate * prior).masked_fill(~option_mask, -1e4)
def diagnostics(self):
return {
"request_gate_mean": float(self.last_gate.mean()),
"request_gate_min": float(self.last_gate.min()),
"request_gate_max": float(self.last_gate.max()),
}
class AnswerAdapter(nn.Module):
def __init__(self, reasoner_width=1024, encoder_width=3840, rank=64):
super().__init__()
self.norm = nn.LayerNorm(reasoner_width)
self.down = nn.Linear(reasoner_width, rank, bias=False)
self.action_score = nn.Linear(rank, encoder_width, bias=False)
nn.init.zeros_(self.action_score.weight)
def forward(self, high):
return self.action_score(F.silu(self.down(self.norm(high))))
class ResidualAnswerReadout(nn.Module):
def __init__(
self, decision, letter, hybrid, reasoner_width=1024, encoder_width=3840, rank=64
):
super().__init__()
self.decision, self.letter, self.hybrid = decision, letter, hybrid
self.answer_adapter = AnswerAdapter(reasoner_width, encoder_width, rank)
if hybrid:
self.prior_gate = nn.Parameter(torch.tensor(math.atanh(0.1)))
self.last_relative_delta = None
def forward(self, hidden, mask, positions, option_mask):
# Capture the existing reasoner's output from its one normal forward.
# A scoped hook avoids duplicating/reimplementing the reader or reasoner.
high = []
handle = self.decision.decision.reasoner.register_forward_hook(
lambda module, inputs, output: high.append(output.high_state)
)
try:
decision = self.decision(hidden, mask, positions, option_mask)
finally:
handle.remove()
if len(high) != 1:
raise ValueError("Expected exactly one reasoner forward")
delta = self.answer_adapter(high.pop())
original = final_state(hidden, mask)
self.last_relative_delta = delta.detach().float().square().mean(
-1
).sqrt() / original.detach().float().square().mean(-1).sqrt().clamp_min(1e-8)
logits = F.linear(
original + delta, self.letter.weight[: option_mask.shape[1]]
).float()
cap = self.letter.softcap
if cap:
logits = cap * (logits / cap).tanh()
if self.hybrid:
logits = decision + self.prior_gate.tanh() * logits
return logits.masked_fill(~option_mask, -1e4)
def diagnostics(self):
values = {
"answer_delta_relative_rms_mean": float(self.last_relative_delta.mean())
}
if self.hybrid:
values["pretrained_prior_coefficient"] = float(
self.prior_gate.detach().tanh()
)
return values
class MixedReadout(RequestGateReadout):
def __init__(
self,
decision,
letter,
encoder_width=3840,
reasoner_width=1024,
gate_width=32,
rank=64,
):
component_rng = torch.get_rng_state()
super().__init__(decision, letter, encoder_width, gate_width)
# Match each component's separate trial initialization. Adding the gate
# must not move the adapter's CPU initialization along the RNG sequence.
with torch.random.fork_rng(devices=[]):
torch.set_rng_state(component_rng)
self.answer_adapter = AnswerAdapter(reasoner_width, encoder_width, rank)
self.last_relative_delta = None
def forward(self, hidden, mask, positions, option_mask):
high = []
handle = self.decision.decision.reasoner.register_forward_hook(
lambda module, inputs, output: high.append(output.high_state)
)
try:
decision = self.decision(hidden, mask, positions, option_mask)
finally:
handle.remove()
if len(high) != 1:
raise ValueError("Expected exactly one reasoner forward")
original = final_state(hidden, mask)
delta = self.answer_adapter(high.pop())
prior = F.linear(
original + delta, self.letter.weight[: option_mask.shape[1]]
).float()
cap = self.letter.softcap
if cap:
prior = cap * (prior / cap).tanh()
prior = prior.masked_fill(~option_mask, -1e4)
state = F.silu(self.request_project(self.request_norm(original)))
stats = torch.cat(
(
confidence_features(decision, option_mask),
confidence_features(prior, option_mask),
option_mask.sum(-1, keepdim=True).float().log(),
),
-1,
)
gate = self.action_score(torch.cat((state, stats), -1)).float().sigmoid()
self.last_gate = gate.detach()
self.last_relative_delta = delta.detach().float().square().mean(
-1
).sqrt() / original.detach().float().square().mean(-1).sqrt().clamp_min(1e-8)
return (decision + gate * prior).masked_fill(~option_mask, -1e4)
def diagnostics(self):
return {
**super().diagnostics(),
"answer_delta_relative_rms_mean": float(self.last_relative_delta.mean()),
}