Download detector.py from Anonymous1611/Lattice_Spike: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/detector.py
- Command line
-
hf download hf://Anonymous1611/Lattice_Spike/detector.py
-
curl -L -o detector.py https://huggingface.co/Anonymous1611/Lattice_Spike/resolve/main/detector.py
15.4 kB
| """ | |
| End-to-end LatticeSpike detector. | |
| Wires together the three layers: | |
| encoding : BoxEncoder + geometric/concretization signals | |
| dynamics : VolumeModulatedRSNN (default) or LSTMDynamics (--ablation lstm) | |
| detection : sliding-window spike density -> grounding burst | |
| The forward pass consumes a batch of LLM-derived inputs and returns spikes, the | |
| per-step burst score rho(t), the box objects, and everything the combined loss | |
| needs. | |
| Input dict (`signals`) keys: | |
| h_box : (B, T, d_model) hidden state for the box encoder | |
| delta : (B, T, L) per-layer KL trace | |
| attn_ent : (B, T) attention entropy | |
| erank : (B, T) effective rank | |
| topk_probs: (B, T, K) top-K next-token probabilities | |
| topk_ids : (B, T, K) top-K next-token ids (into the box-space table) | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import torch | |
| from torch import nn, Tensor | |
| import torch.nn.functional as F | |
| from .boxes import BoxEncoder, BoxBatch, geometric_signals, concretization_residual | |
| from .signals import ( | |
| MRLProjection, | |
| kl_variance, | |
| kl_mean, | |
| assemble_channels, | |
| ChannelStack, | |
| ) | |
| from .rsnn import VolumeModulatedRSNN, RSNNOutput | |
| from .lstm_dyn import LSTMDynamics | |
| class LatticeSpikeOutput: | |
| rho: Tensor # (B, T) spike density | |
| anomaly: Tensor # (B, T) per-token reconstruction-error anomaly score | |
| burst: Tensor # (B, T) boolean grounding-burst flag | |
| box: BoxBatch | |
| channels: ChannelStack | |
| rsnn: RSNNOutput | |
| concretization: Tensor # (B, T) m_t | |
| def score(self) -> Tensor: | |
| """Primary detection score (the paper's unsupervised anomaly signal).""" | |
| return self.anomaly | |
| class LayerMixer(nn.Module): | |
| """ELMo-style scalar mixture over several hidden layers. | |
| h = gamma * sum_k softmax(w)_k * h_k | |
| Chosen over concatenation deliberately: concatenating K layers would take the | |
| box encoder's input from d_model to K*d_model, so any gain would be | |
| confounded with a much larger input projection. This adds K+1 parameters, | |
| leaves the encoder's input width unchanged, and the learned softmax weights | |
| are directly readable as "which layers carry the signal". | |
| """ | |
| def __init__(self, n_layers: int): | |
| super().__init__() | |
| if n_layers < 1: | |
| raise ValueError("n_layers must be positive") | |
| self.n_layers = int(n_layers) | |
| self.weights = nn.Parameter(torch.zeros(self.n_layers)) | |
| self.gamma = nn.Parameter(torch.ones(1)) | |
| def forward(self, h: Tensor) -> Tensor: | |
| # h: (B, T, K, d_model) -> (B, T, d_model) | |
| if h.shape[-2] != self.n_layers: | |
| raise ValueError( | |
| f"h_box has {h.shape[-2]} layers but the model was built for " | |
| f"{self.n_layers}; the signal cache and --box-hidden-layers disagree" | |
| ) | |
| w = torch.softmax(self.weights, dim=0).view(1, 1, self.n_layers, 1) | |
| return self.gamma * (h * w).sum(dim=-2) | |
| def layer_weights(self) -> list[float]: | |
| """Readable mixture weights, for reporting which layers were used.""" | |
| return torch.softmax(self.weights.detach(), dim=0).tolist() | |
| class LatticeSpike(nn.Module): | |
| def __init__( | |
| self, | |
| d_model: int, | |
| vocab_size: int, | |
| d_box: int = 256, | |
| num_layers: int = 28, | |
| mrl_k: int = 32, | |
| n_neurons: int = 512, | |
| window: int = 8, | |
| anomaly_window: int = 1, | |
| rho_star: float = 0.15, | |
| surrogate: str = "atan", | |
| concretization_scale: float = 1.0, | |
| include_concretization_channel: bool = True, | |
| tie_box_token_emb: bool = False, | |
| box_encoder: str = "mlp", | |
| ablation: str | None = None, | |
| lstm_use_volume: bool = False, | |
| lstm_hidden: int | None = None, | |
| token_embedding_table: torch.Tensor | None = None, | |
| score_head: str = "residual", | |
| score_head_hidden: int = 64, | |
| box_hidden_layers: int = 1, | |
| lookback_dim: int = 0, | |
| lookback_k: int = 16, | |
| ): | |
| super().__init__() | |
| self.box_encoder = box_encoder | |
| # >1 means the cache carries a layer axis on h_box and we learn a scalar | |
| # mixture over it before the box encoder sees anything. | |
| self.box_hidden_layers = int(box_hidden_layers) | |
| self.layer_mixer = ( | |
| LayerMixer(self.box_hidden_layers) if self.box_hidden_layers > 1 else None | |
| ) | |
| self.encoder = BoxEncoder(d_model, d_box=d_box, architecture=box_encoder) | |
| self.mrl = MRLProjection(num_layers, k=mrl_k) | |
| # Token embeddings in *box space*. Two modes: | |
| # tie_box_token_emb=False: a frozen random table (synthetic / legacy). | |
| # tie_box_token_emb=True : a learned projection of the LLM's own token | |
| # embeddings, trained jointly so the concretization residual lives in | |
| # the same space as the box. The per-token embeddings are NOT cached; | |
| # they are reconstructed at run time from signals["topk_ids"] and the | |
| # shared LLM embedding table below (each cached row would be ~7 MB). | |
| self.include_concretization_channel = bool(include_concretization_channel) | |
| self.tie_box_token_emb = bool(tie_box_token_emb) | |
| if self.include_concretization_channel and self.tie_box_token_emb: | |
| self.box_token_proj = nn.Linear(d_model, d_box) | |
| if token_embedding_table is None: | |
| raise ValueError( | |
| "tie_box_token_emb=True requires token_embedding_table " | |
| "(the LLM input-embedding matrix used to reconstruct top-k embeddings)" | |
| ) | |
| # Shared, non-trainable LLM embedding table. persistent=False keeps it | |
| # out of checkpoints (it is re-supplied at construction); fp16 halves | |
| # the ~0.5 GB footprint. | |
| self.register_buffer( | |
| "token_embedding_table", | |
| token_embedding_table.detach().to(dtype=torch.float16), | |
| persistent=False, | |
| ) | |
| elif self.include_concretization_channel: | |
| self.box_token_emb = nn.Embedding(vocab_size, d_box) | |
| self.box_token_emb.weight.requires_grad_(False) | |
| # Optional Ch7: per-head Lookback context-vs-generated ratios, compressed | |
| # to lookback_k dims by a learned projection. Follows the MRLProjection | |
| # pattern used for the per-layer KL trace -- keeping the raw ~512 values | |
| # would more than quadruple the RSNN input width and confound any gain | |
| # with a much larger input projection. | |
| self.lookback_dim = int(lookback_dim) | |
| self.lookback_k = int(lookback_k) if self.lookback_dim > 0 else 0 | |
| self.lookback_proj = ( | |
| nn.Linear(self.lookback_dim, self.lookback_k) | |
| if self.lookback_dim > 0 | |
| else None | |
| ) | |
| # Ch1 (2d) + Ch3 (k) + Ch2, Ch4, Ch5a, Ch5b, optional Ch6, optional Ch7. | |
| in_dim = ( | |
| 2 * d_box + mrl_k + 4 | |
| + int(self.include_concretization_channel) | |
| + self.lookback_k | |
| ) | |
| gate_channels = ( | |
| 6 | |
| + int(self.include_concretization_channel) | |
| + int(self.lookback_proj is not None) | |
| ) | |
| # channel slice template is rebuilt each forward (depends on assemble order) | |
| self.ablation = ablation | |
| if ablation is None: | |
| self.rsnn = VolumeModulatedRSNN( | |
| in_dim=in_dim, | |
| n_neurons=n_neurons, | |
| num_channels=gate_channels, | |
| surrogate=surrogate, | |
| ) | |
| elif ablation == "lstm": | |
| self.rsnn = LSTMDynamics( | |
| in_dim=in_dim, n_neurons=n_neurons, num_channels=gate_channels, | |
| use_volume=lstm_use_volume, hidden=lstm_hidden, | |
| ) | |
| else: | |
| raise ValueError(f"unknown ablation {ablation!r}; expected None or 'lstm'") | |
| self.window = window | |
| self.anomaly_window = int(anomaly_window) | |
| if self.anomaly_window <= 0: | |
| raise ValueError("anomaly_window must be positive") | |
| self.concretization_scale = float(concretization_scale) | |
| self.register_buffer("rho_star", torch.tensor(float(rho_star))) | |
| # Anomaly threshold; calibrate from grounded data via `calibrate()`. | |
| self.register_buffer("anomaly_star", torch.tensor(float("inf"))) | |
| # Coupling coefficient for the optional anchored-contraction loss. | |
| # It learns only when that loss has nonzero weight; retaining the | |
| # parameter preserves old checkpoints and anchor-on control runs. | |
| self.gamma_raw = nn.Parameter(torch.tensor(0.0)) | |
| # Scoring readout. "residual" is the original hand-designed statistic -- | |
| # mean squared one-step reconstruction error, zero learned parameters -- | |
| # which is a holdover from the unsupervised design where reconstruction | |
| # error *was* the anomaly signal. Under a supervised ranking objective the | |
| # loss has to shape the whole network indirectly through that fixed | |
| # statistic. "learned" gives the ranking loss a direct path from the full | |
| # membrane state to the score instead of collapsing it with a mean. | |
| if score_head not in {"residual", "learned"}: | |
| raise ValueError("score_head must be 'residual' or 'learned'") | |
| self.score_head = score_head | |
| if score_head == "learned": | |
| hidden = int(score_head_hidden) | |
| if hidden <= 0: | |
| raise ValueError("score_head_hidden must be positive") | |
| # Input is [u(t), residual(t)]: the membrane state plus the original | |
| # statistic, so the learned head can only add to what it already had. | |
| self.score_mlp = nn.Sequential( | |
| nn.Linear(n_neurons + 1, hidden), | |
| nn.GELU(), | |
| nn.Linear(hidden, 1), | |
| ) | |
| def gamma(self) -> Tensor: | |
| return F.softplus(self.gamma_raw) | |
| def spike_density(self, spikes: Tensor) -> Tensor: | |
| """rho(t) = mean over neurons and a trailing window of width W.""" | |
| # mean across neurons -> (B, T); causal moving average of width `window`. | |
| per_t = spikes.mean(dim=-1) # (B, T) | |
| w = self.window | |
| kernel = torch.ones(1, 1, w, device=spikes.device) / w | |
| padded = F.pad(per_t.unsqueeze(1), (w - 1, 0)) # left/causal pad | |
| rho = F.conv1d(padded, kernel).squeeze(1) # (B, T) | |
| return rho | |
| def reconstruction_anomaly(self, rsnn_out) -> Tensor: | |
| """Per-token temporal reconstruction error (paper's anomaly signal). | |
| High when the membrane trajectory departs from the RSNN's learned | |
| one-step prediction of grounded dynamics -- i.e. when the input channels | |
| carry an anomaly. Returned as a (B, T) score, smoothed over the window. | |
| """ | |
| u, u_pred = rsnn_out.u, rsnn_out.u_pred | |
| err = torch.zeros(u.shape[0], u.shape[1], device=u.device, dtype=u.dtype) | |
| err[:, 1:] = (u_pred[:, :-1] - u[:, 1:]).pow(2).mean(dim=-1) # (B, T) | |
| w = self.anomaly_window | |
| kernel = torch.ones(1, 1, w, device=u.device, dtype=u.dtype) / w | |
| padded = F.pad(err.unsqueeze(1), (w - 1, 0)) | |
| return F.conv1d(padded, kernel).squeeze(1) | |
| def calibrate(self, signals: dict, z: float = 4.0): | |
| """Set the anomaly burst threshold from grounded data: mean + z*std.""" | |
| out = self.forward(signals) | |
| a = out.anomaly | |
| self.anomaly_star = (a.mean() + z * a.std()).detach() | |
| return float(self.anomaly_star) | |
| def forward(self, signals: dict) -> LatticeSpikeOutput: | |
| h_box = signals["h_box"] | |
| if self.layer_mixer is not None: | |
| if h_box.ndim != 4: | |
| raise ValueError( | |
| "model expects multi-layer h_box (B,T,K,d) but the cache " | |
| "supplied a single layer; re-extract or drop --box-hidden-layers" | |
| ) | |
| h_box = self.layer_mixer(h_box) | |
| elif h_box.ndim == 4: | |
| raise ValueError( | |
| "signal cache carries multi-layer h_box but the model was built " | |
| "for one layer; pass --box-hidden-layers to match the cache" | |
| ) | |
| box = self.encoder(h_box) # BoxBatch (B,T,d) | |
| # Channels 1 & 2 | |
| leakage, kappa = geometric_signals(box) | |
| # Channels 3 & 4 | |
| delta = signals["delta"] # (B, T, L) | |
| mrl_kl = self.mrl(delta) # (B, T, k) | |
| kl_var = kl_variance(delta) # (B, T) | |
| klm = kl_mean(delta) # (B, T) | |
| # Channel 5 | |
| attn_ent = signals["attn_ent"] # (B, T) | |
| erank = signals["erank"] # (B, T) | |
| # Channel 6 (concretization residual) | |
| if not self.include_concretization_channel: | |
| m = torch.zeros_like(kappa) | |
| elif self.tie_box_token_emb: | |
| # Reconstruct the LLM input embeddings of the top-k ids from the shared | |
| # table (not cached per token). topk_ids: (B,T,K) -> (B,T,K,d_model). | |
| # sigmoid squashes into [0,1]^d, the space the box corners live in. | |
| topk_embs = self.token_embedding_table[signals["topk_ids"]].float() | |
| topk_emb = torch.sigmoid(self.box_token_proj(topk_embs)) | |
| else: | |
| topk_emb = self.box_token_emb(signals["topk_ids"]) # (B, T, K, d) | |
| if self.include_concretization_channel: | |
| m = concretization_residual(box, signals["topk_probs"], topk_emb) | |
| m = m * self.concretization_scale | |
| modulation_scale = box.modulation_scale | |
| ch = assemble_channels( | |
| leakage=leakage, | |
| kappa=kappa, | |
| mrl_kl=mrl_kl, | |
| kl_var=kl_var, | |
| attn_ent=attn_ent, | |
| erank=erank, | |
| concretization=m, | |
| volume=modulation_scale, | |
| kl_mean_=klm, | |
| include_concretization=self.include_concretization_channel, | |
| lookback=( | |
| self.lookback_proj(signals["lookback"].float()) | |
| if self.lookback_proj is not None | |
| else None | |
| ), | |
| ) | |
| # Hand the channel slice map to the RSNN gate the first time. | |
| self.rsnn.channel_slices = ch.slices | |
| rsnn_out = self.rsnn(ch.x, modulation_scale) | |
| rho = self.spike_density(rsnn_out.spikes) | |
| anomaly = self.reconstruction_anomaly(rsnn_out) | |
| if self.score_head == "learned": | |
| residual = anomaly.unsqueeze(-1) # (B, T, 1) | |
| features = torch.cat([rsnn_out.u, residual], dim=-1) # (B, T, n+1) | |
| anomaly = self.score_mlp(features).squeeze(-1) # (B, T) | |
| # A grounding burst is sustained reconstruction anomaly (the unsupervised | |
| # signal) co-occurring with elevated spike density. The LSTM ablation has | |
| # no spikes (rho == 0), so its burst rule is the anomaly threshold alone. | |
| if self.ablation == "lstm": | |
| burst = anomaly > self.anomaly_star | |
| else: | |
| burst = (anomaly > self.anomaly_star) & (rho > self.rho_star) | |
| return LatticeSpikeOutput( | |
| rho=rho, | |
| anomaly=anomaly, | |
| burst=burst, | |
| box=box, | |
| channels=ch, | |
| rsnn=rsnn_out, | |
| concretization=m, | |
| ) | |