Kestrel-34M-instruct / modeling_kestrel.py
lillycham's picture
Upload Kestrel-34M-instruct
ff2813d verified
Raw History Blame Contribute Delete
21.2 kB
"""Hugging Face modeling code for Kestrel (Looped Titans-MAG)."""
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput
from transformers.models.qwen3.modeling_qwen3 import (
Qwen3MLP,
Qwen3RMSNorm,
Qwen3RotaryEmbedding,
apply_rotary_pos_emb,
)
from .configuration_kestrel import KestrelConfig
def _silu_and_grad(x):
sig = torch.sigmoid(x)
silu = x * sig
grad = sig * (1.0 + x * (1.0 - sig))
return silu, grad
class NeuralMemory(nn.Module):
"""Two-layer fast-weight MLP memory updated per chunk via closed-form MSE gradients."""
def __init__(self, hidden_size, memory_hidden_size, chunk_size, heads=1, forget_bias=-2.0, min_forget=0.0):
super().__init__()
self.hidden_size = hidden_size
self.memory_hidden_size = memory_hidden_size
self.chunk_size = chunk_size
self.heads = heads
self.head_dim = hidden_size // heads
self.mem_head_dim = memory_hidden_size // heads
self.forget_bias = forget_bias
self.min_forget = min_forget
self.q_proj = nn.Linear(hidden_size, hidden_size, bias=False)
self.k_proj = nn.Linear(hidden_size, hidden_size, bias=False)
self.v_proj = nn.Linear(hidden_size, hidden_size, bias=False)
self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False)
# Per-head, per-chunk scalar gates: alpha (forget), eta (momentum), theta (inner lr).
self.gate_proj = nn.Linear(hidden_size, 3 * heads, bias=False)
init_shape_w1 = (memory_hidden_size, hidden_size) if heads == 1 else (heads, self.mem_head_dim, self.head_dim)
init_shape_w2 = (hidden_size, memory_hidden_size) if heads == 1 else (heads, self.head_dim, self.mem_head_dim)
self.w1_init = nn.Parameter(torch.empty(*init_shape_w1))
self.w2_init = nn.Parameter(torch.empty(*init_shape_w2))
nn.init.normal_(self.w1_init, std=self.head_dim ** -0.5)
nn.init.normal_(self.w2_init, std=self.mem_head_dim ** -0.5)
def _project_qkv(self, proj, x, bsz, num_chunks):
h = proj(x).view(bsz, num_chunks, self.chunk_size, self.heads, self.head_dim)
h = F.normalize(h, dim=-1).permute(0, 3, 1, 2, 4).contiguous()
return h.view(bsz * self.heads, num_chunks, self.chunk_size, self.head_dim).float()
def _gates(self, chunk_ctx):
"""Per-head forget (alpha), momentum (eta) and inner-lr (theta) gates, shaped (B * H, n, 1, 1)."""
bsz, num_chunks, _ = chunk_ctx.shape
gates = self.gate_proj(chunk_ctx).view(bsz, num_chunks, self.heads, 3).permute(0, 2, 1, 3).contiguous()
gates = gates.view(bsz * self.heads, num_chunks, 3).float()
alpha = (self.min_forget + (1.0 - self.min_forget) * torch.sigmoid(gates[..., 0] + self.forget_bias))[..., None, None]
eta = torch.sigmoid(gates[..., 1])[..., None, None]
theta = 0.05 * torch.sigmoid(gates[..., 2])[..., None, None]
return alpha, eta, theta
def _initial_state(self, bsz):
w1_base = self.w1_init.unsqueeze(0) if self.heads == 1 else self.w1_init
w2_base = self.w2_init.unsqueeze(0) if self.heads == 1 else self.w2_init
w1 = w1_base.float().unsqueeze(0).expand(bsz, -1, -1, -1).reshape(bsz * self.heads, self.mem_head_dim, self.head_dim).clone()
w2 = w2_base.float().unsqueeze(0).expand(bsz, -1, -1, -1).reshape(bsz * self.heads, self.head_dim, self.mem_head_dim).clone()
return w1, w2, torch.zeros_like(w1), torch.zeros_like(w2)
@staticmethod
def _retrieve(qc, w1, w2):
return torch.bmm(F.silu(torch.bmm(qc, w1.transpose(1, 2))), w2.transpose(1, 2))
def _update(self, state, kc, vc, a_i, e_i, t_i):
"""One momentum step on the closed-form gradient of 0.5 / C * ||M(kc) - vc||^2."""
w1, w2, s1, s2 = state
hk = torch.bmm(kc, w1.transpose(1, 2)) # (B * H, C, m_h)
ak, dsilu = _silu_and_grad(hk)
pred = torch.bmm(ak, w2.transpose(1, 2))
err = (pred - vc) * (1.0 / self.chunk_size) # (B * H, C, d_h)
gw2 = torch.bmm(err.transpose(1, 2), ak) # (B * H, d_h, m_h)
dak = torch.bmm(err, w2) * dsilu # (B * H, C, m_h)
gw1 = torch.bmm(dak.transpose(1, 2), kc) # (B * H, m_h, d_h)
s1 = e_i * s1 - t_i * gw1
s2 = e_i * s2 - t_i * gw2
return (1.0 - a_i) * w1 + s1, (1.0 - a_i) * w2 + s2, s1, s2
def forward(self, x, cache=None):
if cache is not None:
return self._forward_cached(x, cache)
bsz, seq_len, dim = x.shape
c = self.chunk_size
pad = (c - seq_len % c) % c
if pad:
x = F.pad(x, (0, 0, 0, pad))
padded_len = x.shape[1]
num_chunks = padded_len // c
q = self._project_qkv(self.q_proj, x, bsz, num_chunks)
k = self._project_qkv(self.k_proj, x, bsz, num_chunks)
v = self._project_qkv(self.v_proj, x, bsz, num_chunks)
# Average chunk hidden state drives scalar forget/momentum/lr gates per head.
alpha, eta, theta = self._gates(x.view(bsz, num_chunks, c, dim).mean(dim=2))
state = self._initial_state(bsz)
outputs = []
for idx in range(num_chunks):
# Retrieve from current memory state (before updating on chunk idx -> causal).
outputs.append(self._retrieve(q[:, idx], state[0], state[1]))
if idx + 1 < num_chunks:
state = self._update(state, k[:, idx], v[:, idx], alpha[:, idx], eta[:, idx], theta[:, idx])
out = torch.cat(outputs, dim=1).view(bsz, self.heads, padded_len, self.head_dim)
out = out.permute(0, 2, 1, 3).contiguous().view(bsz, padded_len, dim)[:, :seq_len].to(x.dtype)
return self.out_proj(out)
def _forward_cached(self, x, cache):
"""Streaming forward for decoding. Gives the same outputs as forward() on the whole sequence.
The cache holds the memory state after the last complete chunk, plus the
hidden states, keys and values of the current incomplete chunk. The memory
updates only when a chunk is complete, as in the parallel path.
"""
bsz, seq_len, dim = x.shape
c = self.chunk_size
if "state" not in cache:
cache["state"] = self._initial_state(bsz)
cache["x"] = x.new_zeros(bsz, 0, dim)
cache["k"] = x.new_zeros(bsz * self.heads, 0, self.head_dim, dtype=torch.float32)
cache["v"] = x.new_zeros(bsz * self.heads, 0, self.head_dim, dtype=torch.float32)
def project(proj):
h = F.normalize(proj(x).view(bsz, seq_len, self.heads, self.head_dim), dim=-1)
return h.transpose(1, 2).reshape(bsz * self.heads, seq_len, self.head_dim).float()
q, k, v = project(self.q_proj), project(self.k_proj), project(self.v_proj)
outputs = []
start = 0
while start < seq_len:
n = min(c - cache["x"].shape[1], seq_len - start)
span = slice(start, start + n)
w1, w2 = cache["state"][:2]
outputs.append(self._retrieve(q[:, span], w1, w2))
cache["x"] = torch.cat([cache["x"], x[:, span]], dim=1)
cache["k"] = torch.cat([cache["k"], k[:, span]], dim=1)
cache["v"] = torch.cat([cache["v"], v[:, span]], dim=1)
if cache["x"].shape[1] == c:
alpha, eta, theta = self._gates(cache["x"].mean(dim=1, keepdim=True))
cache["state"] = self._update(cache["state"], cache["k"], cache["v"], alpha[:, 0], eta[:, 0], theta[:, 0])
cache["x"], cache["k"], cache["v"] = cache["x"][:, :0], cache["k"][:, :0], cache["v"][:, :0]
start += n
out = torch.cat(outputs, dim=1).view(bsz, self.heads, seq_len, self.head_dim)
out = out.transpose(1, 2).reshape(bsz, seq_len, dim).to(x.dtype)
return self.out_proj(out)
class SlidingWindowAttention(nn.Module):
"""GQA with QK-norm, RoPE, and a local sliding causal window."""
def __init__(self, config):
super().__init__()
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.num_kv_groups = self.num_heads // self.num_kv_heads
self.head_dim = config.head_dim
self.window = config.attention_window
self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.head_dim, bias=False)
self.k_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
self.v_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
self.o_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False)
self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps)
def forward(self, x, cos, sin, cache=None):
bsz, seq_len, _ = x.shape
q = self.q_norm(self.q_proj(x).view(bsz, seq_len, self.num_heads, self.head_dim)).transpose(1, 2)
k = self.k_norm(self.k_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim)).transpose(1, 2)
v = self.v_proj(x).view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
q, k = apply_rotary_pos_emb(q, k, cos, sin)
if cache is not None:
# Keys and values are stored after QK-norm and RoPE. RoPE is relative,
# so the cache only needs the last window - 1 positions.
if "k" in cache:
k = torch.cat([cache["k"], k], dim=2)
v = torch.cat([cache["v"], v], dim=2)
keep = self.window - 1
cache["k"] = k[:, :, max(0, k.shape[2] - keep):]
cache["v"] = v[:, :, max(0, v.shape[2] - keep):]
key_len = k.shape[2]
past = key_len - seq_len
if self.num_kv_groups > 1:
k = k.repeat_interleave(self.num_kv_groups, dim=1)
v = v.repeat_interleave(self.num_kv_groups, dim=1)
if past == 0 and self.window >= seq_len:
attn = F.scaled_dot_product_attention(q, k, v, is_causal=True)
elif seq_len == 1 and key_len <= self.window:
attn = F.scaled_dot_product_attention(q, k, v)
else:
pos = torch.arange(key_len, device=x.device)
diff = pos[past:, None] - pos[None, :]
mask = (diff >= 0) & (diff < self.window)
attn = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
return self.o_proj(attn.transpose(1, 2).contiguous().view(bsz, seq_len, -1))
class TitansMAGBlock(nn.Module):
"""Parallel sliding-window attention + neural memory combined via a learned gate."""
def __init__(self, config):
super().__init__()
self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = SlidingWindowAttention(config)
self.attn_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) if getattr(config, "norm_attn_branch", False) else nn.Identity()
self.memory = NeuralMemory(
config.hidden_size, config.memory_hidden_size, config.memory_chunk_size,
heads=getattr(config, "memory_heads", 1),
forget_bias=getattr(config, "memory_forget_bias", -2.0),
min_forget=getattr(config, "memory_min_forget", 0.0),
)
self.mem_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.mag_gate = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.mlp = Qwen3MLP(config)
def forward(self, x, cos, sin, cache=None):
h = self.input_layernorm(x)
attn_cache, mem_cache = (None, None) if cache is None else (cache.setdefault("attn", {}), cache.setdefault("mem", {}))
attn_out = self.attn_norm(self.self_attn(h, cos, sin, attn_cache))
mem_out = self.mem_norm(self.memory(h, mem_cache))
gate = torch.sigmoid(self.mag_gate(h))
x = x + (1.0 - gate) * attn_out + gate * mem_out
x = x + self.mlp(self.post_attention_layernorm(x))
return x
class KestrelForCausalLM(PreTrainedModel):
"""Titans-MAG causal LM with optional prelude/coda and looped recurrent core."""
config_class = KestrelConfig
_tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"}
def __init__(self, config):
super().__init__(config)
self.config = config
self.num_loops = getattr(config, "num_loops", 1)
prelude_layers = getattr(config, "prelude_layers", 0)
coda_layers = getattr(config, "coda_layers", 0)
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=getattr(config, "pad_token_id", None))
if prelude_layers:
self.prelude = nn.ModuleList([TitansMAGBlock(config) for _ in range(prelude_layers)])
self.layers = nn.ModuleList([TitansMAGBlock(config) for _ in range(config.num_hidden_layers)])
if self.num_loops > 1:
self.loop_x0_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.loop_inject = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
# 3D step embedding stays in AdamW (no_decay) rather than Muon.
self.loop_embed = nn.Parameter(torch.empty(self.num_loops, 1, config.hidden_size))
if coda_layers:
self.coda = nn.ModuleList([TitansMAGBlock(config) for _ in range(coda_layers)])
self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = Qwen3RotaryEmbedding(config=config)
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
self.post_init()
def get_input_embeddings(self):
return self.embed_tokens
def get_output_embeddings(self):
return self.lm_head
def forward(self, input_ids, cache=None, logits_to_keep=0, **kwargs):
"""Run the model. With a cache dict, continue from the tokens that it holds.
The cache is a plain dict that this method fills on the first call. Each
block call (prelude, every loop of the core, coda) gets its own entry.
"""
x = self.embed_tokens(input_ids)
offset = 0 if cache is None else cache.get("pos", 0)
pos_ids = torch.arange(offset, offset + input_ids.shape[1], device=input_ids.device).unsqueeze(0)
cos, sin = self.rotary_emb(x, pos_ids)
layer_caches = None if cache is None else iter(cache.setdefault("layers", [{} for _ in range(self._block_calls())]))
def run(layer, x):
return layer(x, cos, sin, None if layer_caches is None else next(layer_caches))
if hasattr(self, "prelude"):
for layer in self.prelude:
x = run(layer, x)
if self.num_loops > 1:
x0_normed = self.loop_x0_norm(x)
for loop_idx in range(self.num_loops):
x = x + self.loop_inject(x0_normed) + self.loop_embed[loop_idx]
for layer in self.layers:
x = run(layer, x)
else:
for layer in self.layers:
x = run(layer, x)
if hasattr(self, "coda"):
for layer in self.coda:
x = run(layer, x)
if cache is not None:
cache["pos"] = offset + input_ids.shape[1]
if logits_to_keep:
x = x[:, -logits_to_keep:]
logits = self.lm_head(self.norm(x))
return CausalLMOutput(logits=logits)
def _block_calls(self):
prelude = len(self.prelude) if hasattr(self, "prelude") else 0
coda = len(self.coda) if hasattr(self, "coda") else 0
return prelude + self.num_loops * len(self.layers) + coda
@torch.no_grad()
def generate(self, input_ids, max_new_tokens=64, do_sample=True, temperature=1.0,
top_k=0, top_p=1.0, repetition_penalty=1.0, eos_token_id=None, bad_words_ids=None,
use_cache=True, num_return_sequences=1, return_logprobs=False, pad_token_id=None, **kwargs):
"""Sample new tokens.
With use_cache, the prompt runs once and each step then feeds only the
newest token. This gives the same logits as the uncached path while the
sequence fits in max_position_embeddings. After that, the cached path
keeps streaming (attention is local and RoPE is relative), but the
uncached path drops the oldest tokens and recomputes the memory.
num_return_sequences samples a group of continuations of one prompt
(input_ids must have batch size 1). The prompt runs once and its cache
is copied for each row. All rows stay at the same position, so their
memory chunks complete on the same step. A row stops at a stop token;
after that it gets pad_token_id (or the first stop token) until all
rows stop.
With return_logprobs, the result is (tokens, logprobs). logprobs has one
value per new token: the log-probability of that token under the raw
model distribution (temperature 1, no top-k, top-p or penalty). It is
0 for padding after a row stops.
"""
stop_list = list(eos_token_id) if isinstance(eos_token_id, (list, tuple, set)) else ([eos_token_id] if eos_token_id is not None else [])
bad_ids = [item[0] for item in bad_words_ids] if bad_words_ids else []
group = num_return_sequences
if group > 1 and input_ids.shape[0] != 1:
raise ValueError("num_return_sequences > 1 needs a batch of one prompt")
fill = pad_token_id if pad_token_id is not None else (stop_list[0] if stop_list else 0)
stop_ids = torch.tensor(stop_list, device=input_ids.device, dtype=input_ids.dtype)
max_ctx = self.config.max_position_embeddings
cache = {} if use_cache else None
tokens = input_ids
if cache is not None:
logits = self(tokens[:, -max_ctx:], cache=cache, logits_to_keep=1).logits[:, -1, :]
if group > 1:
cache = _repeat_batch(cache, group)
logits = logits.repeat(group, 1)
tokens = tokens.repeat(group, 1)
finished = torch.zeros(tokens.shape[0], dtype=torch.bool, device=tokens.device)
logprobs = []
for step in range(max_new_tokens):
if cache is None:
logits = self(tokens[:, -max_ctx:]).logits[:, -1, :]
elif step > 0:
logits = self(tokens[:, -1:], cache=cache, logits_to_keep=1).logits[:, -1, :]
raw = logits.float().log_softmax(dim=-1) if return_logprobs else None
if repetition_penalty != 1.0:
score = torch.gather(logits, 1, tokens)
score = torch.where(score < 0, score * repetition_penalty, score / repetition_penalty)
logits = logits.scatter(1, tokens, score)
if bad_ids:
logits[:, bad_ids] = float("-inf")
if not do_sample:
next_tok = logits.argmax(dim=-1, keepdim=True)
else:
logits = logits / temperature
if top_k > 0:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits = logits.masked_fill(logits < v[:, [-1]], float("-inf"))
if top_p < 1.0:
sorted_logits, sorted_idx = torch.sort(logits, descending=True)
probs = F.softmax(sorted_logits, dim=-1)
remove = torch.cumsum(probs, dim=-1) - probs > top_p
sorted_logits = sorted_logits.masked_fill(remove, float("-inf"))
logits = logits.scatter(1, sorted_idx, sorted_logits)
next_tok = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1)
next_tok = next_tok.masked_fill(finished[:, None], fill)
if return_logprobs:
logprobs.append(raw.gather(1, next_tok).squeeze(1).masked_fill(finished, 0.0))
tokens = torch.cat([tokens, next_tok], dim=-1)
finished = finished | torch.isin(next_tok[:, 0], stop_ids)
if bool(finished.all()):
break
if return_logprobs:
empty = tokens.new_zeros(tokens.shape[0], 0, dtype=torch.float32)
return tokens, torch.stack(logprobs, dim=1) if logprobs else empty
return tokens
def _repeat_batch(cache, n):
"""Copy a batch-1 decode cache n times along the batch dimension.
Memory tensors use a (batch * heads) leading dimension in batch-major order,
so tiling a batch-1 cache with repeat gives the right layout for both kinds.
"""
if isinstance(cache, torch.Tensor):
return cache.repeat(n, *([1] * (cache.ndim - 1)))
if isinstance(cache, dict):
return {key: _repeat_batch(value, n) for key, value in cache.items()}
if isinstance(cache, (list, tuple)):
return type(cache)(_repeat_batch(value, n) for value in cache)
return cache