Lattice_Spike / detector.py
Md Saiful Islam Sajol
Upload Lattice_Spike project with full LFS weights
e2c9d42 verified
Raw History Blame Contribute Delete
15.4 kB
"""
End-to-end LatticeSpike detector.
Wires together the three layers:
encoding : BoxEncoder + geometric/concretization signals
dynamics : VolumeModulatedRSNN (default) or LSTMDynamics (--ablation lstm)
detection : sliding-window spike density -> grounding burst
The forward pass consumes a batch of LLM-derived inputs and returns spikes, the
per-step burst score rho(t), the box objects, and everything the combined loss
needs.
Input dict (`signals`) keys:
h_box : (B, T, d_model) hidden state for the box encoder
delta : (B, T, L) per-layer KL trace
attn_ent : (B, T) attention entropy
erank : (B, T) effective rank
topk_probs: (B, T, K) top-K next-token probabilities
topk_ids : (B, T, K) top-K next-token ids (into the box-space table)
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
from torch import nn, Tensor
import torch.nn.functional as F
from .boxes import BoxEncoder, BoxBatch, geometric_signals, concretization_residual
from .signals import (
MRLProjection,
kl_variance,
kl_mean,
assemble_channels,
ChannelStack,
)
from .rsnn import VolumeModulatedRSNN, RSNNOutput
from .lstm_dyn import LSTMDynamics
@dataclass
class LatticeSpikeOutput:
rho: Tensor # (B, T) spike density
anomaly: Tensor # (B, T) per-token reconstruction-error anomaly score
burst: Tensor # (B, T) boolean grounding-burst flag
box: BoxBatch
channels: ChannelStack
rsnn: RSNNOutput
concretization: Tensor # (B, T) m_t
@property
def score(self) -> Tensor:
"""Primary detection score (the paper's unsupervised anomaly signal)."""
return self.anomaly
class LayerMixer(nn.Module):
"""ELMo-style scalar mixture over several hidden layers.
h = gamma * sum_k softmax(w)_k * h_k
Chosen over concatenation deliberately: concatenating K layers would take the
box encoder's input from d_model to K*d_model, so any gain would be
confounded with a much larger input projection. This adds K+1 parameters,
leaves the encoder's input width unchanged, and the learned softmax weights
are directly readable as "which layers carry the signal".
"""
def __init__(self, n_layers: int):
super().__init__()
if n_layers < 1:
raise ValueError("n_layers must be positive")
self.n_layers = int(n_layers)
self.weights = nn.Parameter(torch.zeros(self.n_layers))
self.gamma = nn.Parameter(torch.ones(1))
def forward(self, h: Tensor) -> Tensor:
# h: (B, T, K, d_model) -> (B, T, d_model)
if h.shape[-2] != self.n_layers:
raise ValueError(
f"h_box has {h.shape[-2]} layers but the model was built for "
f"{self.n_layers}; the signal cache and --box-hidden-layers disagree"
)
w = torch.softmax(self.weights, dim=0).view(1, 1, self.n_layers, 1)
return self.gamma * (h * w).sum(dim=-2)
def layer_weights(self) -> list[float]:
"""Readable mixture weights, for reporting which layers were used."""
return torch.softmax(self.weights.detach(), dim=0).tolist()
class LatticeSpike(nn.Module):
def __init__(
self,
d_model: int,
vocab_size: int,
d_box: int = 256,
num_layers: int = 28,
mrl_k: int = 32,
n_neurons: int = 512,
window: int = 8,
anomaly_window: int = 1,
rho_star: float = 0.15,
surrogate: str = "atan",
concretization_scale: float = 1.0,
include_concretization_channel: bool = True,
tie_box_token_emb: bool = False,
box_encoder: str = "mlp",
ablation: str | None = None,
lstm_use_volume: bool = False,
lstm_hidden: int | None = None,
token_embedding_table: torch.Tensor | None = None,
score_head: str = "residual",
score_head_hidden: int = 64,
box_hidden_layers: int = 1,
lookback_dim: int = 0,
lookback_k: int = 16,
):
super().__init__()
self.box_encoder = box_encoder
# >1 means the cache carries a layer axis on h_box and we learn a scalar
# mixture over it before the box encoder sees anything.
self.box_hidden_layers = int(box_hidden_layers)
self.layer_mixer = (
LayerMixer(self.box_hidden_layers) if self.box_hidden_layers > 1 else None
)
self.encoder = BoxEncoder(d_model, d_box=d_box, architecture=box_encoder)
self.mrl = MRLProjection(num_layers, k=mrl_k)
# Token embeddings in *box space*. Two modes:
# tie_box_token_emb=False: a frozen random table (synthetic / legacy).
# tie_box_token_emb=True : a learned projection of the LLM's own token
# embeddings, trained jointly so the concretization residual lives in
# the same space as the box. The per-token embeddings are NOT cached;
# they are reconstructed at run time from signals["topk_ids"] and the
# shared LLM embedding table below (each cached row would be ~7 MB).
self.include_concretization_channel = bool(include_concretization_channel)
self.tie_box_token_emb = bool(tie_box_token_emb)
if self.include_concretization_channel and self.tie_box_token_emb:
self.box_token_proj = nn.Linear(d_model, d_box)
if token_embedding_table is None:
raise ValueError(
"tie_box_token_emb=True requires token_embedding_table "
"(the LLM input-embedding matrix used to reconstruct top-k embeddings)"
)
# Shared, non-trainable LLM embedding table. persistent=False keeps it
# out of checkpoints (it is re-supplied at construction); fp16 halves
# the ~0.5 GB footprint.
self.register_buffer(
"token_embedding_table",
token_embedding_table.detach().to(dtype=torch.float16),
persistent=False,
)
elif self.include_concretization_channel:
self.box_token_emb = nn.Embedding(vocab_size, d_box)
self.box_token_emb.weight.requires_grad_(False)
# Optional Ch7: per-head Lookback context-vs-generated ratios, compressed
# to lookback_k dims by a learned projection. Follows the MRLProjection
# pattern used for the per-layer KL trace -- keeping the raw ~512 values
# would more than quadruple the RSNN input width and confound any gain
# with a much larger input projection.
self.lookback_dim = int(lookback_dim)
self.lookback_k = int(lookback_k) if self.lookback_dim > 0 else 0
self.lookback_proj = (
nn.Linear(self.lookback_dim, self.lookback_k)
if self.lookback_dim > 0
else None
)
# Ch1 (2d) + Ch3 (k) + Ch2, Ch4, Ch5a, Ch5b, optional Ch6, optional Ch7.
in_dim = (
2 * d_box + mrl_k + 4
+ int(self.include_concretization_channel)
+ self.lookback_k
)
gate_channels = (
6
+ int(self.include_concretization_channel)
+ int(self.lookback_proj is not None)
)
# channel slice template is rebuilt each forward (depends on assemble order)
self.ablation = ablation
if ablation is None:
self.rsnn = VolumeModulatedRSNN(
in_dim=in_dim,
n_neurons=n_neurons,
num_channels=gate_channels,
surrogate=surrogate,
)
elif ablation == "lstm":
self.rsnn = LSTMDynamics(
in_dim=in_dim, n_neurons=n_neurons, num_channels=gate_channels,
use_volume=lstm_use_volume, hidden=lstm_hidden,
)
else:
raise ValueError(f"unknown ablation {ablation!r}; expected None or 'lstm'")
self.window = window
self.anomaly_window = int(anomaly_window)
if self.anomaly_window <= 0:
raise ValueError("anomaly_window must be positive")
self.concretization_scale = float(concretization_scale)
self.register_buffer("rho_star", torch.tensor(float(rho_star)))
# Anomaly threshold; calibrate from grounded data via `calibrate()`.
self.register_buffer("anomaly_star", torch.tensor(float("inf")))
# Coupling coefficient for the optional anchored-contraction loss.
# It learns only when that loss has nonzero weight; retaining the
# parameter preserves old checkpoints and anchor-on control runs.
self.gamma_raw = nn.Parameter(torch.tensor(0.0))
# Scoring readout. "residual" is the original hand-designed statistic --
# mean squared one-step reconstruction error, zero learned parameters --
# which is a holdover from the unsupervised design where reconstruction
# error *was* the anomaly signal. Under a supervised ranking objective the
# loss has to shape the whole network indirectly through that fixed
# statistic. "learned" gives the ranking loss a direct path from the full
# membrane state to the score instead of collapsing it with a mean.
if score_head not in {"residual", "learned"}:
raise ValueError("score_head must be 'residual' or 'learned'")
self.score_head = score_head
if score_head == "learned":
hidden = int(score_head_hidden)
if hidden <= 0:
raise ValueError("score_head_hidden must be positive")
# Input is [u(t), residual(t)]: the membrane state plus the original
# statistic, so the learned head can only add to what it already had.
self.score_mlp = nn.Sequential(
nn.Linear(n_neurons + 1, hidden),
nn.GELU(),
nn.Linear(hidden, 1),
)
@property
def gamma(self) -> Tensor:
return F.softplus(self.gamma_raw)
def spike_density(self, spikes: Tensor) -> Tensor:
"""rho(t) = mean over neurons and a trailing window of width W."""
# mean across neurons -> (B, T); causal moving average of width `window`.
per_t = spikes.mean(dim=-1) # (B, T)
w = self.window
kernel = torch.ones(1, 1, w, device=spikes.device) / w
padded = F.pad(per_t.unsqueeze(1), (w - 1, 0)) # left/causal pad
rho = F.conv1d(padded, kernel).squeeze(1) # (B, T)
return rho
def reconstruction_anomaly(self, rsnn_out) -> Tensor:
"""Per-token temporal reconstruction error (paper's anomaly signal).
High when the membrane trajectory departs from the RSNN's learned
one-step prediction of grounded dynamics -- i.e. when the input channels
carry an anomaly. Returned as a (B, T) score, smoothed over the window.
"""
u, u_pred = rsnn_out.u, rsnn_out.u_pred
err = torch.zeros(u.shape[0], u.shape[1], device=u.device, dtype=u.dtype)
err[:, 1:] = (u_pred[:, :-1] - u[:, 1:]).pow(2).mean(dim=-1) # (B, T)
w = self.anomaly_window
kernel = torch.ones(1, 1, w, device=u.device, dtype=u.dtype) / w
padded = F.pad(err.unsqueeze(1), (w - 1, 0))
return F.conv1d(padded, kernel).squeeze(1)
@torch.no_grad()
def calibrate(self, signals: dict, z: float = 4.0):
"""Set the anomaly burst threshold from grounded data: mean + z*std."""
out = self.forward(signals)
a = out.anomaly
self.anomaly_star = (a.mean() + z * a.std()).detach()
return float(self.anomaly_star)
def forward(self, signals: dict) -> LatticeSpikeOutput:
h_box = signals["h_box"]
if self.layer_mixer is not None:
if h_box.ndim != 4:
raise ValueError(
"model expects multi-layer h_box (B,T,K,d) but the cache "
"supplied a single layer; re-extract or drop --box-hidden-layers"
)
h_box = self.layer_mixer(h_box)
elif h_box.ndim == 4:
raise ValueError(
"signal cache carries multi-layer h_box but the model was built "
"for one layer; pass --box-hidden-layers to match the cache"
)
box = self.encoder(h_box) # BoxBatch (B,T,d)
# Channels 1 & 2
leakage, kappa = geometric_signals(box)
# Channels 3 & 4
delta = signals["delta"] # (B, T, L)
mrl_kl = self.mrl(delta) # (B, T, k)
kl_var = kl_variance(delta) # (B, T)
klm = kl_mean(delta) # (B, T)
# Channel 5
attn_ent = signals["attn_ent"] # (B, T)
erank = signals["erank"] # (B, T)
# Channel 6 (concretization residual)
if not self.include_concretization_channel:
m = torch.zeros_like(kappa)
elif self.tie_box_token_emb:
# Reconstruct the LLM input embeddings of the top-k ids from the shared
# table (not cached per token). topk_ids: (B,T,K) -> (B,T,K,d_model).
# sigmoid squashes into [0,1]^d, the space the box corners live in.
topk_embs = self.token_embedding_table[signals["topk_ids"]].float()
topk_emb = torch.sigmoid(self.box_token_proj(topk_embs))
else:
topk_emb = self.box_token_emb(signals["topk_ids"]) # (B, T, K, d)
if self.include_concretization_channel:
m = concretization_residual(box, signals["topk_probs"], topk_emb)
m = m * self.concretization_scale
modulation_scale = box.modulation_scale
ch = assemble_channels(
leakage=leakage,
kappa=kappa,
mrl_kl=mrl_kl,
kl_var=kl_var,
attn_ent=attn_ent,
erank=erank,
concretization=m,
volume=modulation_scale,
kl_mean_=klm,
include_concretization=self.include_concretization_channel,
lookback=(
self.lookback_proj(signals["lookback"].float())
if self.lookback_proj is not None
else None
),
)
# Hand the channel slice map to the RSNN gate the first time.
self.rsnn.channel_slices = ch.slices
rsnn_out = self.rsnn(ch.x, modulation_scale)
rho = self.spike_density(rsnn_out.spikes)
anomaly = self.reconstruction_anomaly(rsnn_out)
if self.score_head == "learned":
residual = anomaly.unsqueeze(-1) # (B, T, 1)
features = torch.cat([rsnn_out.u, residual], dim=-1) # (B, T, n+1)
anomaly = self.score_mlp(features).squeeze(-1) # (B, T)
# A grounding burst is sustained reconstruction anomaly (the unsupervised
# signal) co-occurring with elevated spike density. The LSTM ablation has
# no spikes (rho == 0), so its burst rule is the anomaly threshold alone.
if self.ablation == "lstm":
burst = anomaly > self.anomaly_star
else:
burst = (anomaly > self.anomaly_star) & (rho > self.rho_star)
return LatticeSpikeOutput(
rho=rho,
anomaly=anomaly,
burst=burst,
box=box,
channels=ch,
rsnn=rsnn_out,
concretization=m,
)