WarpStateAI / warpstate /model.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
15.5 kB
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import List
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from .config import WarpStateConfig
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
self.use_fused = False
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.use_fused and x.device.type == "cuda":
return torch.ops.aten._fused_rms_norm(x, [x.shape[-1]], self.weight, self.eps)[0]
dtype = x.dtype
x32 = x.float()
x32 = x32 * torch.rsqrt(x32.pow(2).mean(dim=-1, keepdim=True) + self.eps)
return (x32.to(dtype) * self.weight.to(dtype))
def rotate_half(x: torch.Tensor) -> torch.Tensor:
x1 = x[..., ::2]
x2 = x[..., 1::2]
return torch.stack((-x2, x1), dim=-1).flatten(-2)
def rope_cos_sin(length: int, dim: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
pos = torch.arange(length, device=device, dtype=torch.float32)
inv = 1.0 / (10000 ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim))
freqs = torch.outer(pos, inv)
emb = torch.repeat_interleave(freqs, 2, dim=-1)
return emb.cos().to(dtype), emb.sin().to(dtype)
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
# x: [B, N, H, C, Dh], cos/sin: [C, Dh]
cos = cos.view(1, 1, 1, cos.shape[0], cos.shape[1])
sin = sin.view(1, 1, 1, sin.shape[0], sin.shape[1])
return x * cos + rotate_half(x) * sin
class SwiGLU(nn.Module):
def __init__(self, dim: int, hidden: int):
super().__init__()
self.up_gate = nn.Linear(dim, hidden * 2, bias=False)
self.down = nn.Linear(hidden, dim, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
up, gate = self.up_gate(x).chunk(2, dim=-1)
return self.down(F.silu(gate) * up)
@dataclass
class LayerCache:
fast_state: torch.Tensor
slow_state: torch.Tensor
k_cache: torch.Tensor
v_cache: torch.Tensor
write_accum: torch.Tensor
chunk_len: int = 0
class WarpStateCore(nn.Module):
"""
GPU-first experimental block.
The local path performs causal attention only inside a fixed-size tile.
The long-range path compresses completed tiles into two constant-size
tensor states (fast + slow). A single fused input projection produces
Q/K/V/gate/U for both paths.
"""
def __init__(self, cfg: WarpStateConfig):
super().__init__()
self.cfg = cfg
d = cfg.d_model
h = cfg.n_heads
self.h = h
self.hd = cfg.head_dim
self.chunk = cfg.chunk_size
self.norm1 = RMSNorm(d, cfg.rms_eps)
self.in_proj = nn.Linear(d, 5 * d, bias=False) # Q K V gate memory-U
self.out_proj = nn.Linear(d, d, bias=False)
self.norm2 = RMSNorm(d, cfg.rms_eps)
self.ffn = SwiGLU(d, cfg.ffn_hidden)
# Per-head timescales. Start with a quick and a slow memory.
self.fast_decay_logit = nn.Parameter(torch.full((h,), math.log(0.90 / 0.10)))
self.slow_decay_logit = nn.Parameter(torch.full((h,), math.log(0.99 / 0.01)))
self.memory_mix_logit = nn.Parameter(torch.zeros(h))
cos, sin = rope_cos_sin(self.chunk, self.hd, torch.device("cpu"), torch.float32)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
def _split(self, z: torch.Tensor) -> tuple[torch.Tensor, ...]:
b, t, _ = z.shape
q, k, v, g, u = self.in_proj(z).chunk(5, dim=-1)
def heads(y: torch.Tensor) -> torch.Tensor:
return y.view(b, t, self.h, self.hd)
return heads(q), heads(k), heads(v), heads(g), heads(u)
def forward(self, x: torch.Tensor) -> torch.Tensor:
b, t, d = x.shape
if t % self.chunk != 0:
raise ValueError(f"sequence length {t} must be divisible by chunk_size={self.chunk}")
n_chunks = t // self.chunk
z = self.norm1(x)
q, k, v, g, u = self._split(z)
# [B, N, H, C, Dh]
def chunkify(y: torch.Tensor) -> torch.Tensor:
return y.view(b, n_chunks, self.chunk, self.h, self.hd).permute(0, 1, 3, 2, 4).contiguous()
q = chunkify(q)
k = chunkify(k)
v = chunkify(v)
g = chunkify(g)
u = chunkify(u)
cos, sin = self.rope_cos.to(q.dtype), self.rope_sin.to(q.dtype)
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
# Local tile path: all tiles are batched together, so this is a single
# large SDPA workload rather than token-by-token attention.
q_local = q.reshape(b * n_chunks, self.h, self.chunk, self.hd)
k_local = k.reshape(b * n_chunks, self.h, self.chunk, self.hd)
v_local = v.reshape(b * n_chunks, self.h, self.chunk, self.hd)
local = F.scaled_dot_product_attention(
q_local,
k_local,
v_local,
dropout_p=self.cfg.dropout if self.training else 0.0,
is_causal=True,
).view(b, n_chunks, self.h, self.chunk, self.hd)
# Long-range tensor memory. Memory read uses only previous completed
# tiles; the current tile is written after its outputs are formed.
fast = torch.zeros((b, self.h, self.hd, self.hd), device=x.device, dtype=q.dtype)
slow = torch.zeros_like(fast)
fast_decay = torch.sigmoid(self.fast_decay_logit).to(q.dtype).view(1, self.h, 1, 1)
slow_decay = torch.sigmoid(self.slow_decay_logit).to(q.dtype).view(1, self.h, 1, 1)
memory_mix = torch.sigmoid(self.memory_mix_logit).to(q.dtype).view(1, self.h, 1, 1)
if self.cfg.parallel_memory:
# A strictly lower-triangular decay matrix reads completed tiles only.
# Batch all writes and reads instead of launching kernels per tile.
writes = (torch.tanh(k).transpose(-2, -1) @ torch.tanh(u)) / self.chunk
index = torch.arange(n_chunks, device=x.device)
lag = index[:, None] - index[None, :] - 1
mask = lag >= 0
def states(logit):
decay = logit.float().sigmoid()[:, None, None]
weights = (1 - decay) * decay.pow(lag.clamp_min(0)[None]) * mask
values = writes.permute(0, 2, 1, 3, 4).flatten(-2)
result = torch.matmul(weights.to(values.dtype), values)
return result.view(b, self.h, n_chunks, self.hd, self.hd).transpose(1, 2)
mix = memory_mix.unsqueeze(1)
state = mix * states(self.fast_decay_logit) + (1 - mix) * states(self.slow_decay_logit)
memory = (q @ state) / math.sqrt(self.hd)
gate = torch.sigmoid(g)
y = gate * local + (1 - gate) * memory
y = y.permute(0, 1, 3, 2, 4).contiguous().view(b, t, d)
x = x + self.out_proj(y)
return x + self.ffn(self.norm2(x))
out_chunks: list[torch.Tensor] = []
for i in range(n_chunks):
qi = q[:, i]
ki = k[:, i]
ui = u[:, i]
gi = torch.sigmoid(g[:, i])
read_fast = torch.matmul(qi, fast) / math.sqrt(self.hd)
read_slow = torch.matmul(qi, slow) / math.sqrt(self.hd)
memory = memory_mix * read_fast + (1.0 - memory_mix) * read_slow
yi = gi * local[:, i] + (1.0 - gi) * memory
out_chunks.append(yi)
# B,H,D,C @ B,H,C,D -> B,H,D,D. tanh bounds the write and makes
# early training considerably more stable than an unbounded outer product.
write = torch.matmul(
torch.tanh(ki).transpose(-2, -1),
torch.tanh(ui),
) / float(self.chunk)
fast = fast_decay * fast + (1.0 - fast_decay) * write
slow = slow_decay * slow + (1.0 - slow_decay) * write
y = torch.stack(out_chunks, dim=1) # [B,N,H,C,Dh]
y = y.permute(0, 1, 3, 2, 4).contiguous().view(b, t, d)
x = x + self.out_proj(y)
x = x + self.ffn(self.norm2(x))
return x
@torch.no_grad()
def init_cache(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> LayerCache:
state_shape = (batch_size, self.h, self.hd, self.hd)
kv_shape = (batch_size, self.h, self.chunk, self.hd)
return LayerCache(
fast_state=torch.zeros(state_shape, device=device, dtype=dtype),
slow_state=torch.zeros(state_shape, device=device, dtype=dtype),
k_cache=torch.zeros(kv_shape, device=device, dtype=dtype),
v_cache=torch.zeros(kv_shape, device=device, dtype=dtype),
write_accum=torch.zeros(state_shape, device=device, dtype=dtype),
chunk_len=0,
)
@torch.no_grad()
def step(self, x: torch.Tensor, cache: LayerCache) -> torch.Tensor:
# x: [B,1,D]
assert x.shape[1] == 1
b = x.shape[0]
z = self.norm1(x)
q, k, v, g, u = self._split(z)
q = q.permute(0, 2, 1, 3) # B,H,1,Dh
k = k.permute(0, 2, 1, 3)
v = v.permute(0, 2, 1, 3)
g = g.permute(0, 2, 1, 3)
u = u.permute(0, 2, 1, 3)
pos = cache.chunk_len
cos, sin = rope_cos_sin(pos + 1, self.hd, x.device, q.dtype)
c = cos[pos : pos + 1].view(1, 1, 1, self.hd)
s = sin[pos : pos + 1].view(1, 1, 1, self.hd)
q = q * c + rotate_half(q) * s
k = k * c + rotate_half(k) * s
cache.k_cache[:, :, pos : pos + 1].copy_(k)
cache.v_cache[:, :, pos : pos + 1].copy_(v)
keys = cache.k_cache[:, :, : pos + 1]
vals = cache.v_cache[:, :, : pos + 1]
local = F.scaled_dot_product_attention(q, keys, vals, is_causal=False, dropout_p=0.0)
fast_decay = torch.sigmoid(self.fast_decay_logit).to(q.dtype).view(1, self.h, 1, 1)
slow_decay = torch.sigmoid(self.slow_decay_logit).to(q.dtype).view(1, self.h, 1, 1)
memory_mix = torch.sigmoid(self.memory_mix_logit).to(q.dtype).view(1, self.h, 1, 1)
read_fast = torch.matmul(q, cache.fast_state) / math.sqrt(self.hd)
read_slow = torch.matmul(q, cache.slow_state) / math.sqrt(self.hd)
memory = memory_mix * read_fast + (1.0 - memory_mix) * read_slow
gate = torch.sigmoid(g)
y = gate * local + (1.0 - gate) * memory
cache.write_accum.add_(torch.matmul(torch.tanh(k).transpose(-2, -1), torch.tanh(u)))
y = y.permute(0, 2, 1, 3).contiguous().view(b, 1, self.cfg.d_model)
x = x + self.out_proj(y)
x = x + self.ffn(self.norm2(x))
if pos + 1 == self.chunk:
write = cache.write_accum / float(self.chunk)
cache.fast_state.mul_(fast_decay).add_(write * (1.0 - fast_decay))
cache.slow_state.mul_(slow_decay).add_(write * (1.0 - slow_decay))
cache.k_cache.zero_()
cache.v_cache.zero_()
cache.write_accum.zero_()
cache.chunk_len = 0
else:
cache.chunk_len = pos + 1
return x
class WarpStateLM(nn.Module):
def __init__(self, cfg: WarpStateConfig):
super().__init__()
cfg.validate()
self.cfg = cfg
self.embed = nn.Embedding(cfg.vocab_size, cfg.d_model)
self.cores = nn.ModuleList([WarpStateCore(cfg) for _ in range(cfg.n_cores)])
# Tiny depth-specific modulation lets shared cores behave differently on
# each recurrent-depth pass without duplicating the large matrices.
self.depth_scale = nn.Parameter(torch.zeros(cfg.logical_depth, cfg.d_model))
self.depth_bias = nn.Parameter(torch.zeros(cfg.logical_depth, cfg.d_model))
self.final_norm = RMSNorm(cfg.d_model, cfg.rms_eps)
self.apply(self._init_weights)
# Smaller residual projections are useful because the same cores are reused.
scale = 1.0 / math.sqrt(2.0 * cfg.logical_depth)
for core in self.cores:
nn.init.normal_(core.out_proj.weight, mean=0.0, std=0.02 * scale)
nn.init.normal_(core.ffn.down.weight, mean=0.0, std=0.02 * scale)
@staticmethod
def _init_weights(module: nn.Module) -> None:
if isinstance(module, (nn.Linear, nn.Embedding)):
nn.init.normal_(module.weight, mean=0.0, std=0.02)
def num_parameters(self) -> int:
return sum(p.numel() for p in self.parameters())
def forward(self, input_ids: torch.Tensor, targets: torch.Tensor | None = None, *, return_logits: bool = True):
x = self.embed(input_ids)
for depth in range(self.cfg.logical_depth):
x = x * (1.0 + self.depth_scale[depth]) + self.depth_bias[depth]
core = self.cores[depth % self.cfg.n_cores]
if self.training and self.cfg.gradient_checkpointing:
x = checkpoint(core, x, use_reentrant=False)
else:
x = core(x)
x = self.final_norm(x)
if targets is not None and not return_logits:
# Recompute each vocabulary projection during backward, so the full
# sequence-by-vocabulary logits and softmax are never retained.
hidden = x.reshape(-1, x.size(-1))
labels = targets.reshape(-1)
def chunk_loss(h, w, y):
return F.cross_entropy(F.linear(h, w).float(), y, reduction="sum")
loss = x.new_zeros((), dtype=torch.float32)
loss_chunk_size = getattr(self, "loss_chunk_size", self.cfg.loss_chunk_size)
for start in range(0, labels.numel(), loss_chunk_size):
stop = start + loss_chunk_size
if self.training and torch.is_grad_enabled():
value = checkpoint(chunk_loss, hidden[start:stop], self.embed.weight,
labels[start:stop], use_reentrant=False)
else:
value = chunk_loss(hidden[start:stop], self.embed.weight, labels[start:stop])
loss = loss + value
# Empty ranks in a final distributed update contribute a finite zero loss.
return None, loss / labels.ne(-100).sum().clamp_min(1)
logits = F.linear(x, self.embed.weight) # tied LM head
if targets is None:
return logits
loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))
return logits, loss
@torch.no_grad()
def init_generation_cache(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> List[LayerCache]:
caches: list[LayerCache] = []
for depth in range(self.cfg.logical_depth):
core = self.cores[depth % self.cfg.n_cores]
caches.append(core.init_cache(batch_size, device, dtype))
return caches
@torch.no_grad()
def step(self, input_ids: torch.Tensor, caches: List[LayerCache]) -> torch.Tensor:
# input_ids [B,1], returns logits [B,1,V]
x = self.embed(input_ids)
for depth in range(self.cfg.logical_depth):
x = x * (1.0 + self.depth_scale[depth]) + self.depth_bias[depth]
core = self.cores[depth % self.cfg.n_cores]
x = core.step(x, caches[depth])
x = self.final_norm(x)
return F.linear(x, self.embed.weight)