""" End-to-end LatticeSpike detector. Wires together the three layers: encoding : BoxEncoder + geometric/concretization signals dynamics : VolumeModulatedRSNN (default) or LSTMDynamics (--ablation lstm) detection : sliding-window spike density -> grounding burst The forward pass consumes a batch of LLM-derived inputs and returns spikes, the per-step burst score rho(t), the box objects, and everything the combined loss needs. Input dict (`signals`) keys: h_box : (B, T, d_model) hidden state for the box encoder delta : (B, T, L) per-layer KL trace attn_ent : (B, T) attention entropy erank : (B, T) effective rank topk_probs: (B, T, K) top-K next-token probabilities topk_ids : (B, T, K) top-K next-token ids (into the box-space table) """ from __future__ import annotations from dataclasses import dataclass import torch from torch import nn, Tensor import torch.nn.functional as F from .boxes import BoxEncoder, BoxBatch, geometric_signals, concretization_residual from .signals import ( MRLProjection, kl_variance, kl_mean, assemble_channels, ChannelStack, ) from .rsnn import VolumeModulatedRSNN, RSNNOutput from .lstm_dyn import LSTMDynamics @dataclass class LatticeSpikeOutput: rho: Tensor # (B, T) spike density anomaly: Tensor # (B, T) per-token reconstruction-error anomaly score burst: Tensor # (B, T) boolean grounding-burst flag box: BoxBatch channels: ChannelStack rsnn: RSNNOutput concretization: Tensor # (B, T) m_t @property def score(self) -> Tensor: """Primary detection score (the paper's unsupervised anomaly signal).""" return self.anomaly class LayerMixer(nn.Module): """ELMo-style scalar mixture over several hidden layers. h = gamma * sum_k softmax(w)_k * h_k Chosen over concatenation deliberately: concatenating K layers would take the box encoder's input from d_model to K*d_model, so any gain would be confounded with a much larger input projection. This adds K+1 parameters, leaves the encoder's input width unchanged, and the learned softmax weights are directly readable as "which layers carry the signal". """ def __init__(self, n_layers: int): super().__init__() if n_layers < 1: raise ValueError("n_layers must be positive") self.n_layers = int(n_layers) self.weights = nn.Parameter(torch.zeros(self.n_layers)) self.gamma = nn.Parameter(torch.ones(1)) def forward(self, h: Tensor) -> Tensor: # h: (B, T, K, d_model) -> (B, T, d_model) if h.shape[-2] != self.n_layers: raise ValueError( f"h_box has {h.shape[-2]} layers but the model was built for " f"{self.n_layers}; the signal cache and --box-hidden-layers disagree" ) w = torch.softmax(self.weights, dim=0).view(1, 1, self.n_layers, 1) return self.gamma * (h * w).sum(dim=-2) def layer_weights(self) -> list[float]: """Readable mixture weights, for reporting which layers were used.""" return torch.softmax(self.weights.detach(), dim=0).tolist() class LatticeSpike(nn.Module): def __init__( self, d_model: int, vocab_size: int, d_box: int = 256, num_layers: int = 28, mrl_k: int = 32, n_neurons: int = 512, window: int = 8, anomaly_window: int = 1, rho_star: float = 0.15, surrogate: str = "atan", concretization_scale: float = 1.0, include_concretization_channel: bool = True, tie_box_token_emb: bool = False, box_encoder: str = "mlp", ablation: str | None = None, lstm_use_volume: bool = False, lstm_hidden: int | None = None, token_embedding_table: torch.Tensor | None = None, score_head: str = "residual", score_head_hidden: int = 64, box_hidden_layers: int = 1, lookback_dim: int = 0, lookback_k: int = 16, ): super().__init__() self.box_encoder = box_encoder # >1 means the cache carries a layer axis on h_box and we learn a scalar # mixture over it before the box encoder sees anything. self.box_hidden_layers = int(box_hidden_layers) self.layer_mixer = ( LayerMixer(self.box_hidden_layers) if self.box_hidden_layers > 1 else None ) self.encoder = BoxEncoder(d_model, d_box=d_box, architecture=box_encoder) self.mrl = MRLProjection(num_layers, k=mrl_k) # Token embeddings in *box space*. Two modes: # tie_box_token_emb=False: a frozen random table (synthetic / legacy). # tie_box_token_emb=True : a learned projection of the LLM's own token # embeddings, trained jointly so the concretization residual lives in # the same space as the box. The per-token embeddings are NOT cached; # they are reconstructed at run time from signals["topk_ids"] and the # shared LLM embedding table below (each cached row would be ~7 MB). self.include_concretization_channel = bool(include_concretization_channel) self.tie_box_token_emb = bool(tie_box_token_emb) if self.include_concretization_channel and self.tie_box_token_emb: self.box_token_proj = nn.Linear(d_model, d_box) if token_embedding_table is None: raise ValueError( "tie_box_token_emb=True requires token_embedding_table " "(the LLM input-embedding matrix used to reconstruct top-k embeddings)" ) # Shared, non-trainable LLM embedding table. persistent=False keeps it # out of checkpoints (it is re-supplied at construction); fp16 halves # the ~0.5 GB footprint. self.register_buffer( "token_embedding_table", token_embedding_table.detach().to(dtype=torch.float16), persistent=False, ) elif self.include_concretization_channel: self.box_token_emb = nn.Embedding(vocab_size, d_box) self.box_token_emb.weight.requires_grad_(False) # Optional Ch7: per-head Lookback context-vs-generated ratios, compressed # to lookback_k dims by a learned projection. Follows the MRLProjection # pattern used for the per-layer KL trace -- keeping the raw ~512 values # would more than quadruple the RSNN input width and confound any gain # with a much larger input projection. self.lookback_dim = int(lookback_dim) self.lookback_k = int(lookback_k) if self.lookback_dim > 0 else 0 self.lookback_proj = ( nn.Linear(self.lookback_dim, self.lookback_k) if self.lookback_dim > 0 else None ) # Ch1 (2d) + Ch3 (k) + Ch2, Ch4, Ch5a, Ch5b, optional Ch6, optional Ch7. in_dim = ( 2 * d_box + mrl_k + 4 + int(self.include_concretization_channel) + self.lookback_k ) gate_channels = ( 6 + int(self.include_concretization_channel) + int(self.lookback_proj is not None) ) # channel slice template is rebuilt each forward (depends on assemble order) self.ablation = ablation if ablation is None: self.rsnn = VolumeModulatedRSNN( in_dim=in_dim, n_neurons=n_neurons, num_channels=gate_channels, surrogate=surrogate, ) elif ablation == "lstm": self.rsnn = LSTMDynamics( in_dim=in_dim, n_neurons=n_neurons, num_channels=gate_channels, use_volume=lstm_use_volume, hidden=lstm_hidden, ) else: raise ValueError(f"unknown ablation {ablation!r}; expected None or 'lstm'") self.window = window self.anomaly_window = int(anomaly_window) if self.anomaly_window <= 0: raise ValueError("anomaly_window must be positive") self.concretization_scale = float(concretization_scale) self.register_buffer("rho_star", torch.tensor(float(rho_star))) # Anomaly threshold; calibrate from grounded data via `calibrate()`. self.register_buffer("anomaly_star", torch.tensor(float("inf"))) # Coupling coefficient for the optional anchored-contraction loss. # It learns only when that loss has nonzero weight; retaining the # parameter preserves old checkpoints and anchor-on control runs. self.gamma_raw = nn.Parameter(torch.tensor(0.0)) # Scoring readout. "residual" is the original hand-designed statistic -- # mean squared one-step reconstruction error, zero learned parameters -- # which is a holdover from the unsupervised design where reconstruction # error *was* the anomaly signal. Under a supervised ranking objective the # loss has to shape the whole network indirectly through that fixed # statistic. "learned" gives the ranking loss a direct path from the full # membrane state to the score instead of collapsing it with a mean. if score_head not in {"residual", "learned"}: raise ValueError("score_head must be 'residual' or 'learned'") self.score_head = score_head if score_head == "learned": hidden = int(score_head_hidden) if hidden <= 0: raise ValueError("score_head_hidden must be positive") # Input is [u(t), residual(t)]: the membrane state plus the original # statistic, so the learned head can only add to what it already had. self.score_mlp = nn.Sequential( nn.Linear(n_neurons + 1, hidden), nn.GELU(), nn.Linear(hidden, 1), ) @property def gamma(self) -> Tensor: return F.softplus(self.gamma_raw) def spike_density(self, spikes: Tensor) -> Tensor: """rho(t) = mean over neurons and a trailing window of width W.""" # mean across neurons -> (B, T); causal moving average of width `window`. per_t = spikes.mean(dim=-1) # (B, T) w = self.window kernel = torch.ones(1, 1, w, device=spikes.device) / w padded = F.pad(per_t.unsqueeze(1), (w - 1, 0)) # left/causal pad rho = F.conv1d(padded, kernel).squeeze(1) # (B, T) return rho def reconstruction_anomaly(self, rsnn_out) -> Tensor: """Per-token temporal reconstruction error (paper's anomaly signal). High when the membrane trajectory departs from the RSNN's learned one-step prediction of grounded dynamics -- i.e. when the input channels carry an anomaly. Returned as a (B, T) score, smoothed over the window. """ u, u_pred = rsnn_out.u, rsnn_out.u_pred err = torch.zeros(u.shape[0], u.shape[1], device=u.device, dtype=u.dtype) err[:, 1:] = (u_pred[:, :-1] - u[:, 1:]).pow(2).mean(dim=-1) # (B, T) w = self.anomaly_window kernel = torch.ones(1, 1, w, device=u.device, dtype=u.dtype) / w padded = F.pad(err.unsqueeze(1), (w - 1, 0)) return F.conv1d(padded, kernel).squeeze(1) @torch.no_grad() def calibrate(self, signals: dict, z: float = 4.0): """Set the anomaly burst threshold from grounded data: mean + z*std.""" out = self.forward(signals) a = out.anomaly self.anomaly_star = (a.mean() + z * a.std()).detach() return float(self.anomaly_star) def forward(self, signals: dict) -> LatticeSpikeOutput: h_box = signals["h_box"] if self.layer_mixer is not None: if h_box.ndim != 4: raise ValueError( "model expects multi-layer h_box (B,T,K,d) but the cache " "supplied a single layer; re-extract or drop --box-hidden-layers" ) h_box = self.layer_mixer(h_box) elif h_box.ndim == 4: raise ValueError( "signal cache carries multi-layer h_box but the model was built " "for one layer; pass --box-hidden-layers to match the cache" ) box = self.encoder(h_box) # BoxBatch (B,T,d) # Channels 1 & 2 leakage, kappa = geometric_signals(box) # Channels 3 & 4 delta = signals["delta"] # (B, T, L) mrl_kl = self.mrl(delta) # (B, T, k) kl_var = kl_variance(delta) # (B, T) klm = kl_mean(delta) # (B, T) # Channel 5 attn_ent = signals["attn_ent"] # (B, T) erank = signals["erank"] # (B, T) # Channel 6 (concretization residual) if not self.include_concretization_channel: m = torch.zeros_like(kappa) elif self.tie_box_token_emb: # Reconstruct the LLM input embeddings of the top-k ids from the shared # table (not cached per token). topk_ids: (B,T,K) -> (B,T,K,d_model). # sigmoid squashes into [0,1]^d, the space the box corners live in. topk_embs = self.token_embedding_table[signals["topk_ids"]].float() topk_emb = torch.sigmoid(self.box_token_proj(topk_embs)) else: topk_emb = self.box_token_emb(signals["topk_ids"]) # (B, T, K, d) if self.include_concretization_channel: m = concretization_residual(box, signals["topk_probs"], topk_emb) m = m * self.concretization_scale modulation_scale = box.modulation_scale ch = assemble_channels( leakage=leakage, kappa=kappa, mrl_kl=mrl_kl, kl_var=kl_var, attn_ent=attn_ent, erank=erank, concretization=m, volume=modulation_scale, kl_mean_=klm, include_concretization=self.include_concretization_channel, lookback=( self.lookback_proj(signals["lookback"].float()) if self.lookback_proj is not None else None ), ) # Hand the channel slice map to the RSNN gate the first time. self.rsnn.channel_slices = ch.slices rsnn_out = self.rsnn(ch.x, modulation_scale) rho = self.spike_density(rsnn_out.spikes) anomaly = self.reconstruction_anomaly(rsnn_out) if self.score_head == "learned": residual = anomaly.unsqueeze(-1) # (B, T, 1) features = torch.cat([rsnn_out.u, residual], dim=-1) # (B, T, n+1) anomaly = self.score_mlp(features).squeeze(-1) # (B, T) # A grounding burst is sustained reconstruction anomaly (the unsupervised # signal) co-occurring with elevated spike density. The LSTM ablation has # no spikes (rho == 0), so its burst rule is the anomaly threshold alone. if self.ablation == "lstm": burst = anomaly > self.anomaly_star else: burst = (anomaly > self.anomaly_star) & (rho > self.rho_star) return LatticeSpikeOutput( rho=rho, anomaly=anomaly, burst=burst, box=box, channels=ch, rsnn=rsnn_out, concretization=m, )