recamplify / recamplify.py
saurav-jha's picture
Add the 11 main-table models of the recAMPLIFY paper
8cd7242 verified
Raw History Blame Contribute Delete
25.7 kB
"""recAMPLIFY: AMPLIFY with recursive refinement and Gated Mixture Recurrence (GMR).
A non-recurrent *stem* and *exit* wrap a weight-tied *recurrent core* applied ``L`` times to
a residue-aligned latent state, giving effective depth at a fixed parameter budget. **GMR**
decodes each sequence from a probability-weighted mixture of the states visited during
recursion, gating each cycle's residual update by a learned survival signal.
Five configurations (see ``README.md``):
============== ================ ==================== =========================
``VARIANT`` ``use_ponder_act`` ``ponder_gate_mode`` description
============== ================ ==================== =========================
baseline False -- fixed ``L`` cycles, plain CE
halt_only True ``none`` halt head alone decides halting
gate_scalar True ``scalar`` gated residual, one channel
gate_lowrank True ``lowrank`` gated residual, ``rank`` channels
gate_perdim True ``perdim`` gated residual, one channel per feature
============== ================ ==================== =========================
Module and config field names are the serialisation contract of the released checkpoints, so
they are kept stable even where a different name would read better; the paper's notation is
used for internal variables and docstrings instead.
"""
from dataclasses import dataclass
from typing import Any, Callable, Optional, Tuple
import torch
from torch import nn
from torch.nn.functional import cross_entropy, rms_norm
from transformers.utils import ModelOutput
from .amplify import (
_HALF_DTYPES,
AMPLIFYConfig,
AMPLIFYPreTrainedModel,
EncoderBlock,
RotaryEmbedding,
_flash_available,
)
_GATE_MODES = ("none", "scalar", "lowrank", "perdim")
_CONTEXT_TYPES = ("zero", "zL0")
@dataclass
class RecMaskedLMOutput(ModelOutput):
"""Output of :class:`RecAMPLIFY`.
Attributes:
loss: Masked-LM cross-entropy on masked positions.
logits: Vocabulary logits, same leading shape as ``input_ids``.
ponder_cost: Per-sequence entropy regulariser ``(B,)``; ``None`` without GMR.
avg_cycles: Expected number of recursive cycles; ``None`` without GMR.
"""
loss: Optional[torch.Tensor] = None
logits: Optional[torch.Tensor] = None
ponder_cost: Optional[torch.Tensor] = None
avg_cycles: Optional[torch.Tensor] = None
class RecAMPLIFYConfig(AMPLIFYConfig):
"""Configuration for :class:`RecAMPLIFY`.
Field names match the original ``AMPLIFYSRMConfig`` because they are the wire format of
every pretrained checkpoint's ``config.json``; that code's ``model_type="amplify_srm"`` is
still accepted.
Attributes:
num_core_layers: TOTAL number of blocks, out of which the stem and exit are carved.
no_recur_layers: Leading blocks run once before the loop (the stem).
no_exit_layers: Trailing blocks run once after the loop (the exit).
L_cycles: Number of recursive cycles.
context_type: ``"zero"`` (context is 0) or ``"zL0"`` (context is ``z_init``).
use_ponder_act: Enable GMR.
ponder_gate_mode: ``"none"`` (halt-only), ``"scalar"``, ``"lowrank"`` or ``"perdim"``.
ponder_gate_rank: Number of gate channels in ``"lowrank"`` mode.
ponder_gate_init_bias: Gate-head bias init; ``sigmoid(4) ~ 0.98`` starts gates open.
ponder_decode_perdim: Weight the mixture per feature instead of by a pooled scalar.
ponder_budget_perchannel: Apply the cost per gate channel, then average.
ponder_act_min_steps: Cycles during which halting is blocked.
ponder_act_epsilon: Cumulative-mass threshold for the early exit.
"""
model_type = "recamplify"
# model_type written by the pre-release code; such config.json files are read as recamplify.
legacy_model_type = "amplify_srm"
def __init__(
self,
num_core_layers: int = 12,
no_recur_layers: int = 2,
no_exit_layers: int = 2,
L_cycles: int = 6,
context_type: str = "zero",
use_ponder_act: bool = False,
ponder_gate_mode: Optional[str] = "none",
ponder_gate_rank: int = 16,
ponder_gate_init_bias: float = 4.0,
ponder_decode_perdim: bool = False,
ponder_budget_perchannel: bool = False,
ponder_act_min_steps: int = 1,
ponder_act_epsilon: float = 1e-3,
**kwargs,
):
# Halt-only checkpoints serialise ponder_gate_mode as JSON null.
ponder_gate_mode = ponder_gate_mode or "none"
if context_type not in _CONTEXT_TYPES:
raise ValueError(f"context_type must be one of {_CONTEXT_TYPES}, got {context_type!r}")
if ponder_gate_mode not in _GATE_MODES:
raise ValueError(f"ponder_gate_mode must be one of {_GATE_MODES}, got {ponder_gate_mode!r}")
if no_recur_layers + no_exit_layers >= num_core_layers:
raise ValueError(
f"no_recur_layers + no_exit_layers ({no_recur_layers}+{no_exit_layers}) must leave at "
f"least one recurrent block out of num_core_layers={num_core_layers}"
)
if use_ponder_act and not 0 <= ponder_act_min_steps < L_cycles:
raise ValueError(f"ponder_act_min_steps must be in [0, L_cycles), got {ponder_act_min_steps}")
if use_ponder_act and ponder_gate_mode != "none" and ponder_act_min_steps < 1:
raise ValueError(f"gated GMR requires ponder_act_min_steps >= 1, got {ponder_act_min_steps}")
# AMPLIFY has an untied decoder: every AMPLIFY and recAMPLIFY checkpoint stores its own
# lm_head.weight. transformers >= 4.57 would otherwise auto-tie it to the input embedding.
if kwargs.get("tie_word_embeddings", False):
raise ValueError("recAMPLIFY never ties word embeddings; tie_word_embeddings must be False")
kwargs["tie_word_embeddings"] = False
# Ignored but accepted so old config.json files load without surprises:
# truncBP_n (always full BPTT), ponder_act_lambda_p / ponder_act_prior_free (the cost is
# always prior-free entropy), ponder_act_use_z (the heads always read dz),
# ponder_gate_decode / ponder_eval_decode (always the weighted mixture), use_memory_init,
# learnable_memory_init, use_carry_norm, vary_l_cycles, use_q_halt, detach_q_halt,
# use_full_ponder_act, use_timestep_embed, dis_max_steps, z_H_init_std.
super().__init__(**kwargs)
self.num_core_layers = num_core_layers
self.no_recur_layers = no_recur_layers
self.no_exit_layers = no_exit_layers
self.L_cycles = L_cycles
self.context_type = context_type
self.use_ponder_act = use_ponder_act
self.ponder_gate_mode = ponder_gate_mode
self.ponder_gate_rank = ponder_gate_rank
self.ponder_gate_init_bias = ponder_gate_init_bias
self.ponder_decode_perdim = ponder_decode_perdim
self.ponder_budget_perchannel = ponder_budget_perchannel
self.ponder_act_min_steps = ponder_act_min_steps
self.ponder_act_epsilon = ponder_act_epsilon
@classmethod
def get_config_dict(cls, *args, **kwargs):
"""Read ``config.json``, accepting the pre-release ``model_type`` as this one."""
config_dict, kwargs = super().get_config_dict(*args, **kwargs)
if config_dict.get("model_type") == cls.legacy_model_type:
config_dict["model_type"] = cls.model_type
return config_dict, kwargs
class RecAMPLIFY(AMPLIFYPreTrainedModel):
"""Recursive AMPLIFY encoder with a masked language model head."""
config_class = RecAMPLIFYConfig
base_model_prefix = "recamplify"
def __init__(self, config: RecAMPLIFYConfig, **kwargs: Any) -> None:
super().__init__(config)
hidden_size = config.hidden_size
nc, nr, ne = config.num_core_layers, config.no_recur_layers, config.no_exit_layers
self.embed_tokens = nn.Embedding(config.vocab_size, hidden_size, padding_idx=config.pad_token_id)
# ONE flat list; the stem and exit are slices of it, not separate modules. This is what
# makes `core_layers.{i}.*` in the pretrained checkpoints resolve.
self.core_layers = nn.ModuleList([EncoderBlock(config) for _ in range(nc)])
self.norm = nn.RMSNorm(hidden_size, config.norm_eps)
self.lm_head = nn.Linear(hidden_size, config.vocab_size)
self.rotary_emb = RotaryEmbedding(config)
self._stem = slice(0, nr)
self._recur = slice(nr, nc - ne)
self._exit = slice(nc - ne, nc)
# Frozen initial latent, broadcast to every token of every sequence. Persistent, so it
# round-trips through the checkpoint.
z_init = torch.empty(1, 1, hidden_size)
nn.init.trunc_normal_(z_init, std=config.decoder_init_range)
self.register_buffer("z_init", z_init)
self.gate_channels = 0
if config.use_ponder_act:
# Built for every GMR variant, including the gated ones where it is never called:
# the gated checkpoints contain halt_head.* and would not load without it.
self.halt_head = nn.Linear(hidden_size, 1)
nn.init.constant_(self.halt_head.bias, -5.0)
# Load-compatibility only. The original filled this with a geometric prior for the
# KL cost; the cost is now always prior-free entropy, so it is never read.
self.register_buffer("ponder_prior", torch.zeros(config.L_cycles))
if config.ponder_gate_mode != "none":
self.gate_channels = {
"scalar": 1,
"lowrank": config.ponder_gate_rank,
"perdim": hidden_size,
}[config.ponder_gate_mode]
self.gate_head = nn.Linear(hidden_size, self.gate_channels)
nn.init.constant_(self.gate_head.bias, float(config.ponder_gate_init_bias))
if config.ponder_gate_mode == "lowrank":
# Row-softmaxed, so each feature is a convex mix of the k channels.
self.gate_assign = nn.Parameter(torch.empty(hidden_size, config.ponder_gate_rank))
nn.init.normal_(self.gate_assign, std=0.02)
# The gated paths never call halt_head, so it receives no gradient and DDP
# aborts on it ("parameters that were not used in producing loss"). Freezing
# it makes DDP skip it while keeping halt_head.* in the state dict, which is
# cheaper than the find_unused_parameters=True the original resorted to.
self.halt_head.requires_grad_(False)
# Re-inits every nn.Linear weight but touches no bias, so the head biases set above
# survive and the head weights land on U(+/-decoder_init_range), as in the original.
self.post_init()
# ------------------------------------------------------------------ public API
def set_num_cycles(self, num_cycles: int) -> None:
"""Set the number of recursive cycles ``L`` used at inference (test-time depth).
GMR variants keep their halting rule: the forced halt moves to the new last cycle, and
sequences may still exit earlier once their continuation mass falls below epsilon.
"""
num_cycles = int(num_cycles)
if self.config.use_ponder_act and num_cycles <= self.config.ponder_act_min_steps:
raise ValueError(
f"num_cycles must exceed ponder_act_min_steps={self.config.ponder_act_min_steps}, got {num_cycles}"
)
self.config.L_cycles = num_cycles
# ------------------------------------------------------------------ helpers
def _run(self, layers: slice, h: torch.Tensor, ctx: Tuple) -> torch.Tensor:
"""Run *h* through ``core_layers[layers]``; ``ctx`` carries the attention arguments."""
attention_mask, cos, sin, max_seqlen, cu_seqlens = ctx
for layer in self.core_layers[layers]:
h, _ = layer(h, attention_mask, cos, sin, False, max_seqlen, cu_seqlens)
return h
@staticmethod
def _sequence_ops(z: torch.Tensor, cu_seqlens: Optional[torch.Tensor]) -> Tuple[int, Callable, Callable]:
"""Build ``(num_sequences, bos, broadcast)`` for the packed or padded layout.
``bos(t)`` gathers each sequence's BOS latent into ``(B, D)``. ``broadcast(v)`` expands a
per-sequence ``(B,)`` or ``(B, C)`` value to a tensor broadcastable against ``z``, zeroing
the packed padding tail.
"""
if cu_seqlens is None:
def bos(t: torch.Tensor) -> torch.Tensor:
return t[:, 0]
def broadcast(v: torch.Tensor) -> torch.Tensor:
return (v.unsqueeze(-1) if v.dim() == 1 else v).unsqueeze(1)
return z.shape[0], bos, broadcast
# cu_seqlens is [0, ..., total_tokens, buf_size]: the trailing entry maps the padding
# tail as a dummy sequence, hence the -2.
num_sequences = cu_seqlens.numel() - 2
starts = cu_seqlens[:num_sequences].long()
counts = cu_seqlens[1 : num_sequences + 1].long() - starts
total = int(counts.sum())
seq_idx = z.new_zeros(z.shape[1], dtype=torch.long)
seq_idx[:total] = torch.repeat_interleave(torch.arange(num_sequences, device=z.device), counts)
real = (torch.arange(z.shape[1], device=z.device) < total).unsqueeze(-1)
def bos(t: torch.Tensor) -> torch.Tensor:
return t[0, starts]
def broadcast(v: torch.Tensor) -> torch.Tensor:
v = v.unsqueeze(-1) if v.dim() == 1 else v
return (v[seq_idx] * real.to(v.dtype)).unsqueeze(0)
return num_sequences, bos, broadcast
def _chan_to_dim(self, v: torch.Tensor) -> torch.Tensor:
"""Map a per-channel value ``(B, C)`` to a per-feature value ``(B, D)``."""
mode = self.config.ponder_gate_mode
if mode == "perdim":
return v
if mode == "scalar":
return v.expand(-1, self.config.hidden_size)
return v.float() @ torch.softmax(self.gate_assign.float(), dim=1).t()
# ------------------------------------------------------------ recursion paths
def _recur_fixed(self, z: torch.Tensor, inj: torch.Tensor, ctx: Tuple) -> torch.Tensor:
"""Fixed-depth recursion: ``z_k = core(z_{k-1} + inj)`` for ``k = 1..L``."""
for _ in range(self.config.L_cycles):
z = self._run(self._recur, z + inj, ctx)
return z
def _recur_halt(
self, z: torch.Tensor, inj: torch.Tensor, ctx: Tuple, bos: Callable, broadcast: Callable, num_seq: int
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Halt-only GMR; returns ``(z_weighted, p)`` with ``p`` of shape ``(B, T)``."""
cfg = self.config
surviving = z.new_ones(num_seq, dtype=torch.float32)
halted = torch.zeros(num_seq, dtype=torch.bool, device=z.device)
z_weighted = torch.zeros_like(z)
bos_prev = None
probs = []
for k in range(cfg.L_cycles):
proposal = self._run(self._recur, z + inj, ctx)
# Freeze halted sequences before the head reads the state.
z = torch.where(broadcast(halted), z, proposal)
# dz: the detached BOS delta. The first cycle has no predecessor, so it feeds the
# raw BOS latent (delta against zero), exactly as the original does.
bos_cur = bos(z).detach()
delta = bos_cur if bos_prev is None else bos_cur - bos_prev
if k == cfg.L_cycles - 1:
lam = surviving.new_ones(num_seq) # forced halt: absorb the remainder
else:
lam = torch.sigmoid(self.halt_head(delta.to(z.dtype))[..., 0].float())
if k < cfg.ponder_act_min_steps:
lam = torch.zeros_like(lam)
p = surviving * lam
surviving = surviving * (1.0 - lam)
if k >= cfg.ponder_act_min_steps:
probs.append(p)
# Raw z, no RMSNorm -- this is the formula the halt-only checkpoint was trained
# under. The gated path below norms instead; the asymmetry is intentional.
z_weighted = z_weighted + broadcast(p.to(z.dtype)) * z
halted = halted | (surviving < cfg.ponder_act_epsilon)
bos_prev = bos_cur
if self._early_exit(halted.all()):
break
return z_weighted, torch.stack(probs, dim=1)
def _recur_gated(
self, z: torch.Tensor, inj: torch.Tensor, ctx: Tuple, bos: Callable, broadcast: Callable, num_seq: int
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Gated GMR; returns ``(z_weighted, p)`` with ``p`` of shape ``(B, T, C)``."""
cfg = self.config
surviving = z.new_ones(num_seq, self.gate_channels, dtype=torch.float32)
z_weighted = torch.zeros_like(z)
bos_prev = None
masses = []
for k in range(cfg.L_cycles):
proposal = self._run(self._recur, z + inj, ctx)
# dz over successive *proposals* -- not over the post-update state.
bos_cur = bos(proposal).detach()
delta = bos_cur if bos_prev is None else bos_cur - bos_prev
gate = torch.sigmoid(self.gate_head(delta.to(z.dtype)).float())
if k < cfg.ponder_act_min_steps:
gate = torch.ones_like(gate) # block closing
if k == cfg.L_cycles - 1:
gate = torch.zeros_like(gate) # force halt
# Survival ENTERING this cycle gates the update, so S_0 = 1 makes cycle 1 a full
# replacement and the forced g=0 at k=L-1 does not freeze the last update.
z = z + broadcast(self._chan_to_dim(surviving)).to(z.dtype) * (proposal - z)
mass = surviving * (1.0 - gate)
if k >= cfg.ponder_act_min_steps:
masses.append(mass)
surviving = surviving * gate
weight = self._chan_to_dim(mass) if cfg.ponder_decode_perdim else mass.mean(dim=1, keepdim=True)
# Stateless RMSNorm (no learned weight), not self.norm.
z_weighted = z_weighted + broadcast(weight).to(z.dtype) * rms_norm(
z, (z.shape[-1],), eps=cfg.norm_eps
)
bos_prev = bos_cur
if self._early_exit((surviving.max(dim=1).values < cfg.ponder_act_epsilon).all()):
break
return z_weighted, torch.stack(masses, dim=1)
@staticmethod
def _early_exit(condition: torch.Tensor) -> bool:
"""Whether every sequence has fully halted, so the remaining cycles can be skipped.
Reading the condition costs a device sync per cycle and, under ``torch.compile``, a graph
break, so the check is skipped while compiling and all ``L`` cycles run. That is NOT
equivalent to the early exit: the residual mass (< epsilon) the exit would discard is
instead absorbed by the forced halt at the last cycle, and the extra cycles enter the
halting distribution, which shifts the entropy cost and ``avg_cycles`` slightly.
"""
if torch.compiler.is_compiling():
return False
return bool(condition)
# ------------------------------------------------------------------- ponder
def _ponder(self, p: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
"""Entropy cost ``(B,)`` and expected cycle count from the halting mass ``p``.
Args:
p: ``(B, T)`` for halt-only or ``(B, T, C)`` for gated, over the cycles where
halting was allowed (``T = steps_done - min_steps``).
"""
cfg = self.config
gated = p.dim() == 3
if not gated:
p = p.unsqueeze(-1)
num_cycles = p.shape[1]
# Expected depth always uses the channel-averaged distribution, 1-indexed from
# min_steps + 1 since earlier cycles cannot receive mass.
pooled = p.mean(dim=-1)
pooled = pooled / pooled.sum(dim=1, keepdim=True).clamp(min=1e-8)
steps = torch.arange(1, num_cycles + 1, device=p.device, dtype=pooled.dtype)
avg_cycles = (pooled * (steps + cfg.ponder_act_min_steps)).sum(dim=1).mean()
if not cfg.ponder_budget_perchannel:
p = p.mean(dim=-1, keepdim=True)
# Halt-only uses the raw mass, as the original does; it sums to 1 unless the epsilon
# early exit dropped a residual (< epsilon). The gated cost renormalises.
if gated:
p = p / p.sum(dim=1, keepdim=True).clamp(min=1e-8)
# Negative entropy: penalises collapsing all mass onto a single cycle.
cost = (p * p.clamp(min=1e-8).log()).sum(dim=1)
return cost.mean(dim=-1), avg_cycles.detach()
# ------------------------------------------------------------------ forward
def forward(
self,
input_ids: torch.Tensor,
labels: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
cu_seqlens: Optional[torch.Tensor] = None,
max_seqlen: Optional[int] = None,
**kwargs,
) -> RecMaskedLMOutput:
"""Forward pass for recursive masked language modeling.
Args:
input_ids: Token IDs -- ``(B, S)`` padded or ``(1, T)`` packed.
labels: MLM labels with ``-100`` at unmasked positions.
position_ids: Position IDs, same shape as ``input_ids``.
attention_mask: ``(B, S)`` boolean mask for padded mode.
cu_seqlens: Cumulative sequence lengths for packed mode (``B + 2`` entries).
max_seqlen: Longest sequence in the packed batch.
Returns:
:class:`RecMaskedLMOutput` with loss, logits, ponder_cost and avg_cycles.
"""
cfg = self.config
hidden_states = self.embed_tokens(input_ids)
# Packed mode: validate the FlashAttention requirements and flatten position_ids.
if cu_seqlens is not None:
if position_ids is None:
raise ValueError("Packed sequences require explicit position_ids derived from cu_seqlens.")
if not _flash_available:
raise ValueError("Packed sequences require FlashAttention.")
if not hidden_states.is_cuda:
raise ValueError("Packed sequences require CUDA.")
if not (
hidden_states.dtype in _HALF_DTYPES
or (torch.is_autocast_enabled() and torch.get_autocast_gpu_dtype() in _HALF_DTYPES)
):
raise ValueError("Packed sequences require half precision (autocast or explicit).")
position_ids = position_ids.to(device=input_ids.device, dtype=torch.long).squeeze(0)
# Padded mode: reshape the mask for SDPA and build position IDs if absent.
else:
if attention_mask is not None:
attention_mask = attention_mask.unsqueeze(1).unsqueeze(1).bool()
if position_ids is not None:
position_ids = position_ids.to(device=input_ids.device, dtype=torch.long).expand(
input_ids.shape[0], -1
)
else:
position_ids = torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)
position_ids = position_ids.unsqueeze(0).expand(input_ids.shape[0], -1)
cos, sin = self.rotary_emb(hidden_states, position_ids)
if cu_seqlens is None:
# SDPA needs a head broadcast dimension: (B, S, D/2) -> (B, S, 1, D/2).
cos = cos.unsqueeze(2).to(hidden_states.dtype)
sin = sin.unsqueeze(2).to(hidden_states.dtype)
ctx = (attention_mask, cos, sin, max_seqlen, cu_seqlens)
# Stem runs once; its output is re-injected at every cycle.
stem_out = self._run(self._stem, hidden_states, ctx)
z_0 = self.z_init.expand_as(stem_out)
context = z_0 if cfg.context_type == "zL0" else torch.zeros_like(stem_out)
injection = stem_out + context
ponder_cost, avg_cycles = None, None
if not cfg.use_ponder_act:
z = self._recur_fixed(z_0, injection, ctx)
else:
num_seq, bos, broadcast = self._sequence_ops(stem_out, cu_seqlens)
recur = self._recur_gated if cfg.ponder_gate_mode != "none" else self._recur_halt
z, p = recur(z_0, injection, ctx, bos, broadcast, num_seq)
ponder_cost, avg_cycles = self._ponder(p)
# Read-out: one more core application, then the exit blocks. Note it injects only the
# context, NOT stem_out -- so with context_type="zero" this is core(z) alone.
features = self._run(self._recur, z + context, ctx)
features = self._run(self._exit, features, ctx)
logits = self.lm_head(self.norm(features))
loss = None
if labels is not None:
# Mean CE over masked positions, and 0 (not NaN) when a batch has none, as in the
# original. Dividing a sum by the count avoids a host sync on `labels.any()`.
labels = labels.view(-1)
loss = cross_entropy(logits.view(-1, logits.shape[-1]).float(), labels, reduction="sum")
loss = loss / (labels != -100).sum().clamp(min=1)
return RecMaskedLMOutput(
loss=loss,
logits=logits,
ponder_cost=ponder_cost,
avg_cycles=avg_cycles,
)